Skip to content
Snippets Groups Projects
Commit ba76af66 authored by Dominique Blouin's avatar Dominique Blouin
Browse files

change SQL query, XML parser

parent 8fbc85ca
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
crawler/bin/client.jar
+ 0
− 0
View file @ ba76af66
No preview for this file type
crawler/bin/crawler.jar
+ 0
− 0
View file @ ba76af66
No preview for this file type
crawler/bin/derby.log
+ 1
− 1
View file @ ba76af66
Mon Jun 08 06:56:06 CEST 2015 : Could not listen on port 1527 on host 127.0.0.1:
Mon Jun 08 12:05:31 CEST 2015 : Could not listen on port 1527 on host 127.0.0.1:
java.net.BindException: Address already in use
An exception was thrown during network server startup. DRDA_ListenPort.S:Could not listen on port 1527 on host 127.0.0.1:
java.net.BindException: Address already in use
......
crawler/crawler.exe
+ 1
− 1
View file @ ba76af66
cd bin
java -jar crawler.jar -Djavax.net.ssl.keyStore=mySrvKeystore -Djavax.net.ssl.keyStorePassword=123456-jar
java -Djavax.net.ssl.keyStore=ServerKeyStore -Djavax.net.ssl.keyStorePassword=123456 -jar crawler.jar
crawler/src/web/crawler/Database_query.java
+ 12
− 12
View file @ ba76af66
......@@ -99,27 +99,27 @@ private web.crawler.Database_creation database;
/* =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
/* Construct query */
/* =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
/* Select columes from the tables */
querySQL = "SELECT SOFTWARES.CVE_ID, SOFTWARES.\"NAME\", VULNERABILITIES.SUMMARY \n"
+ "FROM ROOT.SOFTWARES \n\tINNER JOIN ROOT.VULNERABILITIES \n"
+ "\t\tON SOFTWARES.CVE_ID = VULNERABILITIES.CVE_ID \n";
/* Select columes from the tables */
querySQL = "SELECT SOFTWARES.CVE_ID, SOFTWARES.\"NAME\", VULNERABILITIES.SUMMARY "
+ "FROM ROOT.SOFTWARES INNER JOIN ROOT.VULNERABILITIES "
+ "ON SOFTWARES.CVE_ID = VULNERABILITIES.CVE_ID ";
/* Including the arguments in the query */
querySQL += "WHERE VULNERABILITIES.SUMMARY LIKE ? "
+ "AND SOFTWARES.CVE_ID LIKE ? "
+ "AND SOFTWARES.\"NAME\" LIKE ? \n"
+ "AND VULNERABILITIES.CVE_ID LIKE ? "
+ "AND SOFTWARES.\"NAME\" LIKE ? "
+ "FETCH FIRST ? ROWS ONLY";
System.out.println(querySQL);
PreparedStatement prep = this.database.getconn().prepareStatement(querySQL);
//System.out.println(querySQL);
PreparedStatement prep = this.database.getconn().prepareStatement(querySQL);
prep.setString(1, "%"+query.get(0)+"%");
prep.setString(2, "%"+query.get(1)+"%");
prep.setString(3, "%"+query.get(2)+"%");
prep.setInt(4,Integer.valueOf(query.get(3)));
rs = prep.executeQuery();
/* =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
//String xmlcontent = StoreResultsInFile(rs);
......@@ -156,9 +156,9 @@ private web.crawler.Database_creation database;
+ "\tINNER JOIN ROOT.VULNERABILITIES \n"
+ "\t\tON REFERENCESS.CVE_ID = VULNERABILITIES.CVE_ID \n";
/* Including the keywords in the query */
querySQL += "WHERE SOFTWARES.CVE_ID LIKE ?";
querySQL += "WHERE VULNERABILITIES.CVE_ID LIKE ?";
System.out.println(querySQL);
//System.out.println(querySQL);
PreparedStatement prep = this.database.getconn().prepareStatement(querySQL);
......
crawler/src/web/crawler/File_management.java
+ 44
− 16
View file @ ba76af66
......@@ -182,13 +182,13 @@ public class File_management {
// System.out.println("Root element :" + doc.getDocumentElement().getNodeName());
NodeList nList = doc.getElementsByTagName("entry");
for (int temp = 0; temp < nList.getLength(); temp++) {
Node nNode = nList.item(temp);
if (nNode.getNodeType() == Node.ELEMENT_NODE) {
Element eElement = (Element) nNode;
//System.out.println("Vuln id : " + eElement.getAttribute("id"));
......@@ -247,17 +247,12 @@ public class File_management {
} else {
list_cwe_id.add("not available");
}
if (eElement.getElementsByTagName("vuln:product").item(0) != null) {
list_soft_name.add(eElement.getElementsByTagName("vuln:product").item(0).getTextContent());
} else {
list_cwe_id.add("not available");
}
NodeList nList1 = eElement.getElementsByTagName("vuln:references");
for (int i = 0; i < nList1.getLength(); i++) {
list_ref_cve.add(eElement.getAttribute("id"));
list_soft_cve.add(eElement.getAttribute("id"));
if (eElement.getElementsByTagName("vuln:references").item(i) != null) {
list_ref_type.add(eElement.getElementsByTagName("vuln:references").item(i).getAttributes().getNamedItem("reference_type").getNodeValue());
......@@ -278,16 +273,42 @@ public class File_management {
}
}
NodeList nList2 = eElement.getElementsByTagName("vuln:product");
for (int j = 0; j < nList2.getLength(); j++) {
list_soft_cve.add(eElement.getAttribute("id"));
if (eElement.getElementsByTagName("vuln:product").item(j) != null) {
list_soft_name.add(eElement.getElementsByTagName("vuln:product").item(j).getTextContent());
} else {
list_soft_name.add("not available");
}
}
list_pub_date.add(eElement.getElementsByTagName("vuln:published-datetime").item(0).getTextContent());
list_mod_date.add(eElement.getElementsByTagName("vuln:last-modified-datetime").item(0).getTextContent());
list_sum.add(eElement.getElementsByTagName("vuln:summary").item(0).getTextContent());
}
}
System.out.println(list_id.size());
System.out.println(list_ref_type.size());
System.out.println(list_soft_name.size());
System.out.println(list_id.get(list_id.size()-1));
System.out.println(list_ref_cve.get(list_ref_cve.size()-1));
System.out.println(list_soft_cve.get(list_soft_cve.size()-1));
PreparedStatement preparedStmt = database.getconn().prepareStatement("INSERT INTO VULNERABILITIES(CVE_ID,PUB_DATE,MOD_DATE,SCORE,ACCESS_VECTOR,ACCESS_COMPLEXITY,AUTHENTICATION,CONFIDENTIALITY_IMPACT,INTEGRITY_IMPACT,AVAILABILITY_IMPACT,GEN_DATE,CWE_ID,SUMMARY)" + "VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? , ?, ?)");
PreparedStatement preparedStmt1 = database.getconn().prepareStatement("INSERT INTO REFERENCESS(CVE_ID,REF_TYPE,SOURCE,LINK)" + "VALUES ( ?, ?, ?, ?)");
PreparedStatement preparedStmt2 = database.getconn().prepareStatement("INSERT INTO SOFTWARES(CVE_ID,NAME)" + "VALUES ( ?, ?)");
System.out.println("Inserting " + list_id.size() + " data into VULNERABILITIES table ...");
for (int i = 0; i < list_id.size(); i++) {
PreparedStatement preparedStmt = database.getconn().prepareStatement("INSERT INTO VULNERABILITIES(CVE_ID,PUB_DATE,MOD_DATE,SCORE,ACCESS_VECTOR,ACCESS_COMPLEXITY,AUTHENTICATION,CONFIDENTIALITY_IMPACT,INTEGRITY_IMPACT,AVAILABILITY_IMPACT,GEN_DATE,CWE_ID,SUMMARY)" + "VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? , ?, ?)");
preparedStmt.setString(1, list_id.get(i));
preparedStmt.setString(2, list_pub_date.get(i));
preparedStmt.setString(3, list_mod_date.get(i));
......@@ -303,25 +324,32 @@ public class File_management {
preparedStmt.setString(13, list_sum.get(i));
preparedStmt.executeUpdate();
}
System.out.println("Inserting " + list_ref_type.size() + " data into REFERENCESS table ...");
for (int d = 0; d < list_ref_type.size(); d++) {
PreparedStatement preparedStmt1 = database.getconn().prepareStatement("INSERT INTO REFERENCESS(CVE_ID,REF_TYPE,SOURCE,LINK)" + "VALUES ( ?, ?, ?, ?)");
preparedStmt1.setString(1, list_ref_cve.get(d));
preparedStmt1.setString(2, list_ref_type.get(d));
preparedStmt1.setString(3, list_ref_source.get(d));
preparedStmt1.setString(4, list_ref_link.get(d));
preparedStmt1.executeUpdate();
}
System.out.println("Inserting " + list_soft_name.size() + " data into SOFTWARES table ...");
for (int f = 0; f < list_soft_name.size(); f++) {
PreparedStatement preparedStmt2 = database.getconn().prepareStatement("INSERT INTO SOFTWARES(CVE_ID,NAME)" + "VALUES ( ?, ?)");
preparedStmt2.setString(1, list_soft_cve.get(f));
preparedStmt2.setString(2, list_soft_name.get(f));
preparedStmt2.executeUpdate();
}
System.out.println("Records inserted in the database: " + list_id.size());
preparedStmt.close();
preparedStmt1.close();
preparedStmt2.close();
System.out.println("Number of vulnerabilities inserted in the database: " + list_id.size());
System.out.println();
database.setTotalRecordsInDatabase(database.getTotalRecordsInDatabase() + list_id.size());
} catch (ParserConfigurationException | SAXException | IOException | DOMException | SQLException | NumberFormatException e) {
......
crawler/src/web/crawler/WebCrawler.java
+ 3
− 2
View file @ ba76af66
......@@ -34,9 +34,10 @@ public class WebCrawler {
public static void main(String[] args) throws ClassNotFoundException, InstantiationException, IllegalAccessException, SQLException, IOException, AWTException, Exception {
String FileNames[] = {
"nvdcve-2.0-2012.xml",
"nvdcve-2.0-2013.xml",
"nvdcve-2.0-2014.xml",
"nvdcve-2.0-2015.xml"
"nvdcve-2.0-2014.xml"
// "nvdcve-2.0-2015.xml"
};
//Database_creation database = new Database_creation();
database = new Database_creation();
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment