[java] OMDb scraper 08-15-2017, 05:25 AM
#1
This program attempts to scrape the OMDb to identify movies. It extracts the title, year, genre, plot and movie cover. It then outputs this to XML so it can be used by MiniDLNA or Kodi.
Note: To successfully identify movies they must be named properly, for example: "Fight Club (1999).mp4"
Note: To successfully identify movies they must be named properly, for example: "Fight Club (1999).mp4"
Code:
import java.net.HttpURLConnection;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import javax.xml.*;
import java.io.*;
public class OMDb
{
public static String title, plot, year, genre, poster,
clean_title, clean_year, clean_ext,
original_path, original_title,
xml_ext = "nfo",
folder = "folder.jpg",
cover_path, xml_path;
public static String APP_NAME = "OMDb scraper";
public static String[] allowed_file_ext = {
"avi", "m4v", "mp4", "mkv" };
public static void main(String[] args)
{
if( args.length > 0 )
{
if( new File ( args[0] ).exists() ) {
scan_directory( args[0] );
} else {
output( "You need to specify a valid scan directory" );
}
}
else {
output("Usage: " +
"omdb.jar <scan_directory> ");
}
}
public static Integer scan_directory( String path )
{
try
{
output( "Scanning directory: " + path );
File directory = new File( path);
File[] files = directory.listFiles();
for (File file : files)
{
if (file.isDirectory() ) {
scan_directory( file.toString() );
}
else
{
if( is_mime_allowed( file ) == 0
&& parse_details( file.getCanonicalPath() ) == 0 )
{
output( "Looking up metadata for: " + original_title );
int results = do_search( clean_title, clean_year );
if( results == 0 )
{
if( save_poster( poster, cover_path ) == 0 && output_xml( xml_path) == 0 ) {
output( "Successfully saved metadata for: " + file.getCanonicalPath());
}
}
else if ( results == 1 ) {
output( "Error finding metadata for: " + file.getCanonicalPath());
}
}
}
}
} catch (IOException e) {
return 1;
}
return 0;
}
public static Integer is_mime_allowed( File file )
{
String mime = get_extension( file );
for( String allowed_mime : allowed_file_ext )
{
if( allowed_mime.equals( mime ) ) {
return 0;
}
}
return 1;
}
public static Integer parse_details( String title )
{
File file = new File( title );
original_title= file.getName();
original_path = file.getPath().replace( original_title, "");
String[] tokens = original_title.split("\\.(?=[^\\.]+$)");
if( tokens[1] != null )
{
clean_ext = tokens[1];
Pattern pattern = Pattern.compile( "\\((\\d{4})\\)" );
Matcher matcher = pattern.matcher(original_title);
if (matcher.find() )
{
clean_title = tokens[0].replace( matcher.group( 0 ), "" );
clean_year = matcher.group( 1 );
if( is_info_needed() != 0 ) {
return 3;
}
}
else {
return 2;
}
}
else {
return 1;
}
return 0;
}
public static Integer is_info_needed ( )
{
cover_path = original_path + original_title.replace( clean_ext, "") + folder;
xml_path = original_path + original_title.replace( clean_ext, xml_ext);
if( !new File (cover_path).exists() ||
!new File( xml_path ).exists() ) {
return 0;
}
return 1;
}
public static Integer do_search( String title, String year )
{
String url = String.format(
"http://www.omdbapi.com/?t=%s&y=%s&plot=full&r=xml",
title.replaceAll(" ", "+"), year);
String data = request_url(url, null);
if (data != null ) {
return( process_results( data ));
}
return 1;
}
public static Integer save_poster( String url, String file )
{
if( request_url( url, file ) == null ) {
return 0;
}
return 1;
}
public static Integer output_xml( String file )
{
try
{
DocumentBuilderFactory builderFactory = DocumentBuilderFactory.newInstance();
DocumentBuilder builder = builderFactory.newDocumentBuilder();
TransformerFactory transformerFactory = TransformerFactory.newInstance();
Transformer transformer = transformerFactory.newTransformer();
Document doc = builder.newDocument();
Element root = doc.createElement( "movie" );
doc.appendChild( root );
Element element = doc.createElement("title");
element.appendChild(doc.createTextNode(title));
root.appendChild( element );
element = doc.createElement("plot");
element.appendChild(doc.createTextNode(plot));
root.appendChild( element );
element = doc.createElement("capturedate");
element.appendChild(doc.createTextNode(year));
root.appendChild( element );
element = doc.createElement("genre");
element.appendChild(doc.createTextNode(genre));
root.appendChild( element );
transformer.transform( new DOMSource( doc ), new StreamResult( new File( file ) )
);
}
catch ( ParserConfigurationException | TransformerException e ) {
return 1;
}
return 0;
}
public static String request_url( String url, String file )
{
StringBuilder sb = new StringBuilder();
try
{
URL read = new URL(url);
HttpURLConnection conn = (HttpURLConnection) read.openConnection();
int response = conn.getResponseCode();
if (response == 201 || response == 200 )
{
InputStream inputStream = new BufferedInputStream(conn.getInputStream());
if( file != null )
{
OutputStream outputStream = new FileOutputStream(file);
byte[] inputLine = new byte[1024];
int count;
while ((count = inputStream.read( inputLine ) ) != -1) {
outputStream.write(inputLine, 0, count );
}
outputStream.close();
}
else
{
BufferedReader br = new BufferedReader(new InputStreamReader(inputStream));
String inputLine;
while ((inputLine = br.readLine()) != null) {
sb.append(inputLine);
}
inputStream.close();
}
conn.disconnect();
if (!sb.toString().isEmpty()) {
return sb.toString();
}
}
}
catch (IOException e) {
return null;
}
return null;
}
public static Integer process_results( String results )
{
try
{
DocumentBuilderFactory builderFactory = DocumentBuilderFactory.newInstance();
DocumentBuilder builder = builderFactory.newDocumentBuilder();
Document doc = builder.parse( new ByteArrayInputStream( results.getBytes(StandardCharsets.UTF_8 ) ) );
XPathFactory xpathFactory = XPathFactory.newInstance();
XPath xpath = xpathFactory.newXPath();
if( xpath.evaluate( "/root/movie/response/text()", doc, XPathConstants.STRING ).equals( "True" ) )
{
title = (String) xpath.evaluate( "/root/movie/title/text()", doc, XPathConstants.STRING );
plot = (String) xpath.evaluate( "/root/movie/plot/text()", doc, XPathConstants.STRING );
year = (String) xpath.evaluate( "/root/movie/year/text()", doc, XPathConstants.STRING );
genre = (String) xpath.evaluate( "/root/movie/genre/text()", doc, XPathConstants.STRING );
poster = (String) xpath.evaluate( "/root/movie/poster/text()", doc, XPathConstants.STRING );
}
else {
return 2;
}
}
catch ( IOException | ParserConfigurationException | SAXException | XPathExpressionException e ) {
return 1;
}
return 0;
}
private static String get_extension(File file) {
return file.getName().substring( file.getName().lastIndexOf(".") + 1 );
}
public static void output( String message ) {
System.out.println( APP_NAME + "<: " + message );
}
}
(This post was last modified: 08-15-2017, 06:57 AM by titbang.
Edit Reason: code cleanup
)
![[+]](https://sinister.li/images/modern/collapse_collapsed.png)
