Extract categories, description, and thumbnail from rendered HTML pages

This commit is contained in:
Markus Koch 2020-04-30 18:29:53 +02:00
parent 9bdb4095a7
commit 3d0afc953e
2 changed files with 52 additions and 16 deletions

View File

@ -22,22 +22,9 @@ for line in $data; do
fi
if [[ "$title" != "" && "$coords" != "" ]]; then
echo "{\"type\": \"Feature\", \"properties\": {\"name\": \"$title\","
echo -n "\"categories\": ["
urltitle=`echo "$title" | sed 's/ /%20/g'`
cjson=`curl -s "https://wiki.linux-forks.de/mediawiki/api.php?action=query&format=json&titles=$urltitle&prop=categories" | json_reformat -m`
categories=`echo "$cjson" | sed -n 's/.*\("categories":[^]]\+\).*/\1/p'`;
IFS='}'
fc="true";
for entry in $categories; do
if [[ "$fc" != "true" ]]; then
echo -n ",";
fi
category=`echo "$entry" | sed -n 's/.*Category:\([^"]\+\).*/\1/p'`
echo -n "\"$category\"";
fc="false";
done;
IFS=$'\n'
echo "]},";
urltitle=`echo "$title" | sed 's/ /_/g'`
./get_rendered_meta.sh "https://wiki.linux-forks.de/mediawiki/index.php/$urltitle"
echo "},";
echo "\"geometry\": {\"type\": \"Point\", \"coordinates\": [$coords]}},";
title="";
coords="";

View File

@ -0,0 +1,49 @@
#!/bin/bash
BASE="https://wiki.linux-forks.de"
mode="find_description"
thumbnail=""
categories=""
data=`curl -s "$1"`
temp=`echo "$data" | sed -n 's/<p>\(.\+\).*/\1/p' | head -n1`
if [ "$temp" != "" ]; then
temp=`echo "$temp" | sed "s#href=\"#href=\"$BASE#g" | sed 's/"/\\\\"/g' | sed 's/\t//g'`
description="$temp"
mode="find_infobox";
fi
IFS=$'>';
for line in $data; do
if [ "$mode" == "find_infobox" ]; then
if [ "`echo \"$line\" | grep 'infobox'`" != "" ]; then
mode="image";
fi
elif [ "$mode" == "image" ]; then
temp=`echo "$line" | sed -n 's/.*img.*src="\([^"]\+\).*/\1/p'`;
if [ "$temp" != "" ]; then
thumbnail="$BASE$temp"
mode="find_cat"
fi
elif [ "$mode" == "find_cat" ]; then
if [ "`echo \"$line\" | grep 'mw-normal-catlinks'`" != "" ]; then
mode="cat";
fi
elif [ "$mode" == "cat" ]; then
temp=`echo "$line" | sed -n 's/.*title="Category:\([^"]\+\).*/\1/pg' | grep -v 'page does not exist'`
if [ "$temp" != "" ]; then
if [ "$categories" != "" ]; then
categories="$categories,"
fi
categories="$categories\"$temp\""
fi
fi
done
IFS=" ";
echo "\"categories\": [$categories],"
echo "\"image\": \"$thumbnail\","
echo "\"description\": \"$description\""