Extract categories, description, and thumbnail from rendered HTML pages
This commit is contained in:
		
							parent
							
								
									9bdb4095a7
								
							
						
					
					
						commit
						3d0afc953e
					
				@ -22,22 +22,9 @@ for line in $data; do
 | 
			
		||||
	fi
 | 
			
		||||
	if [[ "$title" != "" && "$coords" != "" ]]; then
 | 
			
		||||
		echo "{\"type\": \"Feature\", \"properties\": {\"name\": \"$title\","
 | 
			
		||||
		echo -n "\"categories\": ["
 | 
			
		||||
		urltitle=`echo "$title" | sed 's/ /%20/g'`
 | 
			
		||||
		cjson=`curl -s "https://wiki.linux-forks.de/mediawiki/api.php?action=query&format=json&titles=$urltitle&prop=categories" | json_reformat -m`
 | 
			
		||||
		categories=`echo "$cjson" | sed -n 's/.*\("categories":[^]]\+\).*/\1/p'`;
 | 
			
		||||
		IFS='}'
 | 
			
		||||
		fc="true";
 | 
			
		||||
		for entry in $categories; do
 | 
			
		||||
			if [[ "$fc" != "true" ]]; then
 | 
			
		||||
				echo -n ",";
 | 
			
		||||
			fi
 | 
			
		||||
			category=`echo "$entry" | sed -n 's/.*Category:\([^"]\+\).*/\1/p'`
 | 
			
		||||
			echo -n "\"$category\"";
 | 
			
		||||
			fc="false";
 | 
			
		||||
		done;
 | 
			
		||||
		IFS=$'\n'
 | 
			
		||||
		echo "]},";
 | 
			
		||||
		urltitle=`echo "$title" | sed 's/ /_/g'`
 | 
			
		||||
		./get_rendered_meta.sh "https://wiki.linux-forks.de/mediawiki/index.php/$urltitle"
 | 
			
		||||
		echo "},";
 | 
			
		||||
		echo "\"geometry\": {\"type\": \"Point\", \"coordinates\": [$coords]}},";
 | 
			
		||||
		title="";
 | 
			
		||||
		coords="";
 | 
			
		||||
 | 
			
		||||
							
								
								
									
										49
									
								
								scripts/geojson/get_rendered_meta.sh
									
									
									
									
									
										Executable file
									
								
							
							
						
						
									
										49
									
								
								scripts/geojson/get_rendered_meta.sh
									
									
									
									
									
										Executable file
									
								
							@ -0,0 +1,49 @@
 | 
			
		||||
#!/bin/bash
 | 
			
		||||
 | 
			
		||||
BASE="https://wiki.linux-forks.de"
 | 
			
		||||
 | 
			
		||||
mode="find_description"
 | 
			
		||||
 | 
			
		||||
thumbnail=""
 | 
			
		||||
categories=""
 | 
			
		||||
 | 
			
		||||
data=`curl -s "$1"`
 | 
			
		||||
 | 
			
		||||
temp=`echo "$data" | sed -n 's/<p>\(.\+\).*/\1/p' | head -n1`
 | 
			
		||||
if [ "$temp" != "" ]; then
 | 
			
		||||
	temp=`echo "$temp" | sed "s#href=\"#href=\"$BASE#g" | sed 's/"/\\\\"/g' | sed 's/\t//g'`
 | 
			
		||||
	description="$temp"
 | 
			
		||||
	mode="find_infobox";
 | 
			
		||||
fi
 | 
			
		||||
 | 
			
		||||
IFS=$'>';
 | 
			
		||||
for line in $data; do
 | 
			
		||||
	if [ "$mode" == "find_infobox" ]; then
 | 
			
		||||
		if [ "`echo \"$line\" | grep 'infobox'`" != "" ]; then
 | 
			
		||||
			mode="image";
 | 
			
		||||
		fi
 | 
			
		||||
	elif [ "$mode" == "image" ]; then
 | 
			
		||||
		temp=`echo "$line" | sed -n 's/.*img.*src="\([^"]\+\).*/\1/p'`;
 | 
			
		||||
		if [ "$temp" != "" ]; then
 | 
			
		||||
			thumbnail="$BASE$temp"
 | 
			
		||||
			mode="find_cat"
 | 
			
		||||
		fi
 | 
			
		||||
	elif [ "$mode" == "find_cat" ]; then
 | 
			
		||||
		if [ "`echo \"$line\" | grep 'mw-normal-catlinks'`" != "" ]; then
 | 
			
		||||
			mode="cat";
 | 
			
		||||
		fi
 | 
			
		||||
	elif [ "$mode" == "cat" ]; then
 | 
			
		||||
		temp=`echo "$line" | sed -n 's/.*title="Category:\([^"]\+\).*/\1/pg' | grep -v 'page does not exist'`
 | 
			
		||||
		if [ "$temp" != "" ]; then
 | 
			
		||||
			if [ "$categories" != "" ]; then
 | 
			
		||||
				categories="$categories,"
 | 
			
		||||
			fi
 | 
			
		||||
			categories="$categories\"$temp\""
 | 
			
		||||
		fi
 | 
			
		||||
	fi
 | 
			
		||||
done
 | 
			
		||||
IFS=" ";
 | 
			
		||||
 | 
			
		||||
echo "\"categories\": [$categories],"
 | 
			
		||||
echo "\"image\": \"$thumbnail\","
 | 
			
		||||
echo "\"description\": \"$description\""
 | 
			
		||||
		Loading…
	
	
			
			x
			
			
		
	
		Reference in New Issue
	
	Block a user