Extract categories, description, and thumbnail from rendered HTML pages
This commit is contained in:
		
							parent
							
								
									9bdb4095a7
								
							
						
					
					
						commit
						3d0afc953e
					
				@ -22,22 +22,9 @@ for line in $data; do
 | 
				
			|||||||
	fi
 | 
						fi
 | 
				
			||||||
	if [[ "$title" != "" && "$coords" != "" ]]; then
 | 
						if [[ "$title" != "" && "$coords" != "" ]]; then
 | 
				
			||||||
		echo "{\"type\": \"Feature\", \"properties\": {\"name\": \"$title\","
 | 
							echo "{\"type\": \"Feature\", \"properties\": {\"name\": \"$title\","
 | 
				
			||||||
		echo -n "\"categories\": ["
 | 
							urltitle=`echo "$title" | sed 's/ /_/g'`
 | 
				
			||||||
		urltitle=`echo "$title" | sed 's/ /%20/g'`
 | 
							./get_rendered_meta.sh "https://wiki.linux-forks.de/mediawiki/index.php/$urltitle"
 | 
				
			||||||
		cjson=`curl -s "https://wiki.linux-forks.de/mediawiki/api.php?action=query&format=json&titles=$urltitle&prop=categories" | json_reformat -m`
 | 
							echo "},";
 | 
				
			||||||
		categories=`echo "$cjson" | sed -n 's/.*\("categories":[^]]\+\).*/\1/p'`;
 | 
					 | 
				
			||||||
		IFS='}'
 | 
					 | 
				
			||||||
		fc="true";
 | 
					 | 
				
			||||||
		for entry in $categories; do
 | 
					 | 
				
			||||||
			if [[ "$fc" != "true" ]]; then
 | 
					 | 
				
			||||||
				echo -n ",";
 | 
					 | 
				
			||||||
			fi
 | 
					 | 
				
			||||||
			category=`echo "$entry" | sed -n 's/.*Category:\([^"]\+\).*/\1/p'`
 | 
					 | 
				
			||||||
			echo -n "\"$category\"";
 | 
					 | 
				
			||||||
			fc="false";
 | 
					 | 
				
			||||||
		done;
 | 
					 | 
				
			||||||
		IFS=$'\n'
 | 
					 | 
				
			||||||
		echo "]},";
 | 
					 | 
				
			||||||
		echo "\"geometry\": {\"type\": \"Point\", \"coordinates\": [$coords]}},";
 | 
							echo "\"geometry\": {\"type\": \"Point\", \"coordinates\": [$coords]}},";
 | 
				
			||||||
		title="";
 | 
							title="";
 | 
				
			||||||
		coords="";
 | 
							coords="";
 | 
				
			||||||
 | 
				
			|||||||
							
								
								
									
										49
									
								
								scripts/geojson/get_rendered_meta.sh
									
									
									
									
									
										Executable file
									
								
							
							
						
						
									
										49
									
								
								scripts/geojson/get_rendered_meta.sh
									
									
									
									
									
										Executable file
									
								
							@ -0,0 +1,49 @@
 | 
				
			|||||||
 | 
					#!/bin/bash
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					BASE="https://wiki.linux-forks.de"
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					mode="find_description"
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					thumbnail=""
 | 
				
			||||||
 | 
					categories=""
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					data=`curl -s "$1"`
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					temp=`echo "$data" | sed -n 's/<p>\(.\+\).*/\1/p' | head -n1`
 | 
				
			||||||
 | 
					if [ "$temp" != "" ]; then
 | 
				
			||||||
 | 
						temp=`echo "$temp" | sed "s#href=\"#href=\"$BASE#g" | sed 's/"/\\\\"/g' | sed 's/\t//g'`
 | 
				
			||||||
 | 
						description="$temp"
 | 
				
			||||||
 | 
						mode="find_infobox";
 | 
				
			||||||
 | 
					fi
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					IFS=$'>';
 | 
				
			||||||
 | 
					for line in $data; do
 | 
				
			||||||
 | 
						if [ "$mode" == "find_infobox" ]; then
 | 
				
			||||||
 | 
							if [ "`echo \"$line\" | grep 'infobox'`" != "" ]; then
 | 
				
			||||||
 | 
								mode="image";
 | 
				
			||||||
 | 
							fi
 | 
				
			||||||
 | 
						elif [ "$mode" == "image" ]; then
 | 
				
			||||||
 | 
							temp=`echo "$line" | sed -n 's/.*img.*src="\([^"]\+\).*/\1/p'`;
 | 
				
			||||||
 | 
							if [ "$temp" != "" ]; then
 | 
				
			||||||
 | 
								thumbnail="$BASE$temp"
 | 
				
			||||||
 | 
								mode="find_cat"
 | 
				
			||||||
 | 
							fi
 | 
				
			||||||
 | 
						elif [ "$mode" == "find_cat" ]; then
 | 
				
			||||||
 | 
							if [ "`echo \"$line\" | grep 'mw-normal-catlinks'`" != "" ]; then
 | 
				
			||||||
 | 
								mode="cat";
 | 
				
			||||||
 | 
							fi
 | 
				
			||||||
 | 
						elif [ "$mode" == "cat" ]; then
 | 
				
			||||||
 | 
							temp=`echo "$line" | sed -n 's/.*title="Category:\([^"]\+\).*/\1/pg' | grep -v 'page does not exist'`
 | 
				
			||||||
 | 
							if [ "$temp" != "" ]; then
 | 
				
			||||||
 | 
								if [ "$categories" != "" ]; then
 | 
				
			||||||
 | 
									categories="$categories,"
 | 
				
			||||||
 | 
								fi
 | 
				
			||||||
 | 
								categories="$categories\"$temp\""
 | 
				
			||||||
 | 
							fi
 | 
				
			||||||
 | 
						fi
 | 
				
			||||||
 | 
					done
 | 
				
			||||||
 | 
					IFS=" ";
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					echo "\"categories\": [$categories],"
 | 
				
			||||||
 | 
					echo "\"image\": \"$thumbnail\","
 | 
				
			||||||
 | 
					echo "\"description\": \"$description\""
 | 
				
			||||||
		Loading…
	
	
			
			x
			
			
		
	
		Reference in New Issue
	
	Block a user