diff --git a/crawler/bin/client.jar b/crawler/bin/client.jar
index 355dcf45c8c5210f3c6a76934bdb52eb1312e81a..4285912653b34b89c8d0dd27d690eb3532307ee3 100644
Binary files a/crawler/bin/client.jar and b/crawler/bin/client.jar differ
diff --git a/crawler/bin/crawler.jar b/crawler/bin/crawler.jar
index a15cae656992f7d86ef7371c4639b6ddd7cf2732..ba7b380b2f4df3393a3e6b79f711224685c4d804 100755
Binary files a/crawler/bin/crawler.jar and b/crawler/bin/crawler.jar differ
diff --git a/crawler/bin/derby.log b/crawler/bin/derby.log
index 9bc5b8ba66f01273525ca5239385ea8a5fb7ae89..4b7bc14fb34ecfb1f5ca4d773ece31a8ad2ac711 100755
--- a/crawler/bin/derby.log
+++ b/crawler/bin/derby.log
@@ -1,4 +1,4 @@
-Mon Jun 08 06:56:06 CEST 2015 : Could not listen on port 1527 on host 127.0.0.1:
+Mon Jun 08 12:05:31 CEST 2015 : Could not listen on port 1527 on host 127.0.0.1:
java.net.BindException: Address already in use
An exception was thrown during network server startup. DRDA_ListenPort.S:Could not listen on port 1527 on host 127.0.0.1:
java.net.BindException: Address already in use
diff --git a/crawler/crawler.exe b/crawler/crawler.exe
index b15ea27a250d6f0520772aaaf4270044406951e0..6ec9a7fbe19c5b1ebd6f14417439d03c2b84cdb9 100755
--- a/crawler/crawler.exe
+++ b/crawler/crawler.exe
@@ -1,2 +1,2 @@
cd bin
-java -jar crawler.jar -Djavax.net.ssl.keyStore=mySrvKeystore -Djavax.net.ssl.keyStorePassword=123456-jar
+java -Djavax.net.ssl.keyStore=ServerKeyStore -Djavax.net.ssl.keyStorePassword=123456 -jar crawler.jar
diff --git a/crawler/src/web/crawler/Database_query.java b/crawler/src/web/crawler/Database_query.java
index 318431b0ce26e20a5951c218ac1dfcb064fd4c91..9f68fe8d22ea87e45721c766c94f27b000ead619 100644
--- a/crawler/src/web/crawler/Database_query.java
+++ b/crawler/src/web/crawler/Database_query.java
@@ -99,27 +99,27 @@ private web.crawler.Database_creation database;
/* =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
/* Construct query */
/* =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
- /* Select columes from the tables */
- querySQL = "SELECT SOFTWARES.CVE_ID, SOFTWARES.\"NAME\", VULNERABILITIES.SUMMARY \n"
- + "FROM ROOT.SOFTWARES \n\tINNER JOIN ROOT.VULNERABILITIES \n"
- + "\t\tON SOFTWARES.CVE_ID = VULNERABILITIES.CVE_ID \n";
+
+ /* Select columes from the tables */
+ querySQL = "SELECT SOFTWARES.CVE_ID, SOFTWARES.\"NAME\", VULNERABILITIES.SUMMARY "
+ + "FROM ROOT.SOFTWARES INNER JOIN ROOT.VULNERABILITIES "
+ + "ON SOFTWARES.CVE_ID = VULNERABILITIES.CVE_ID ";
/* Including the arguments in the query */
querySQL += "WHERE VULNERABILITIES.SUMMARY LIKE ? "
- + "AND SOFTWARES.CVE_ID LIKE ? "
- + "AND SOFTWARES.\"NAME\" LIKE ? \n"
+ + "AND VULNERABILITIES.CVE_ID LIKE ? "
+ + "AND SOFTWARES.\"NAME\" LIKE ? "
+ "FETCH FIRST ? ROWS ONLY";
-
- System.out.println(querySQL);
- PreparedStatement prep = this.database.getconn().prepareStatement(querySQL);
+ //System.out.println(querySQL);
+ PreparedStatement prep = this.database.getconn().prepareStatement(querySQL);
prep.setString(1, "%"+query.get(0)+"%");
prep.setString(2, "%"+query.get(1)+"%");
prep.setString(3, "%"+query.get(2)+"%");
prep.setInt(4,Integer.valueOf(query.get(3)));
rs = prep.executeQuery();
-
+
/* =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
//String xmlcontent = StoreResultsInFile(rs);
@@ -156,9 +156,9 @@ private web.crawler.Database_creation database;
+ "\tINNER JOIN ROOT.VULNERABILITIES \n"
+ "\t\tON REFERENCESS.CVE_ID = VULNERABILITIES.CVE_ID \n";
/* Including the keywords in the query */
- querySQL += "WHERE SOFTWARES.CVE_ID LIKE ?";
+ querySQL += "WHERE VULNERABILITIES.CVE_ID LIKE ?";
- System.out.println(querySQL);
+ //System.out.println(querySQL);
PreparedStatement prep = this.database.getconn().prepareStatement(querySQL);
diff --git a/crawler/src/web/crawler/File_management.java b/crawler/src/web/crawler/File_management.java
index 6b7b7cf1dc07c43b784a4c5d047d910cf6ff6db9..01c5194062dba0aee29f2fd2707101edfe60daeb 100644
--- a/crawler/src/web/crawler/File_management.java
+++ b/crawler/src/web/crawler/File_management.java
@@ -182,13 +182,13 @@ public class File_management {
// System.out.println("Root element :" + doc.getDocumentElement().getNodeName());
NodeList nList = doc.getElementsByTagName("entry");
-
+
for (int temp = 0; temp < nList.getLength(); temp++) {
Node nNode = nList.item(temp);
if (nNode.getNodeType() == Node.ELEMENT_NODE) {
-
+
Element eElement = (Element) nNode;
//System.out.println("Vuln id : " + eElement.getAttribute("id"));
@@ -247,17 +247,12 @@ public class File_management {
} else {
list_cwe_id.add("not available");
}
- if (eElement.getElementsByTagName("vuln:product").item(0) != null) {
- list_soft_name.add(eElement.getElementsByTagName("vuln:product").item(0).getTextContent());
- } else {
- list_cwe_id.add("not available");
- }
-
+
NodeList nList1 = eElement.getElementsByTagName("vuln:references");
+
for (int i = 0; i < nList1.getLength(); i++) {
list_ref_cve.add(eElement.getAttribute("id"));
- list_soft_cve.add(eElement.getAttribute("id"));
if (eElement.getElementsByTagName("vuln:references").item(i) != null) {
list_ref_type.add(eElement.getElementsByTagName("vuln:references").item(i).getAttributes().getNamedItem("reference_type").getNodeValue());
@@ -278,16 +273,42 @@ public class File_management {
}
}
+
+ NodeList nList2 = eElement.getElementsByTagName("vuln:product");
+
+ for (int j = 0; j < nList2.getLength(); j++) {
+ list_soft_cve.add(eElement.getAttribute("id"));
+
+ if (eElement.getElementsByTagName("vuln:product").item(j) != null) {
+ list_soft_name.add(eElement.getElementsByTagName("vuln:product").item(j).getTextContent());
+ } else {
+ list_soft_name.add("not available");
+ }
+ }
+
list_pub_date.add(eElement.getElementsByTagName("vuln:published-datetime").item(0).getTextContent());
list_mod_date.add(eElement.getElementsByTagName("vuln:last-modified-datetime").item(0).getTextContent());
list_sum.add(eElement.getElementsByTagName("vuln:summary").item(0).getTextContent());
}
}
-
+
+ System.out.println(list_id.size());
+ System.out.println(list_ref_type.size());
+ System.out.println(list_soft_name.size());
+ System.out.println(list_id.get(list_id.size()-1));
+ System.out.println(list_ref_cve.get(list_ref_cve.size()-1));
+ System.out.println(list_soft_cve.get(list_soft_cve.size()-1));
+
+ PreparedStatement preparedStmt = database.getconn().prepareStatement("INSERT INTO VULNERABILITIES(CVE_ID,PUB_DATE,MOD_DATE,SCORE,ACCESS_VECTOR,ACCESS_COMPLEXITY,AUTHENTICATION,CONFIDENTIALITY_IMPACT,INTEGRITY_IMPACT,AVAILABILITY_IMPACT,GEN_DATE,CWE_ID,SUMMARY)" + "VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? , ?, ?)");
+ PreparedStatement preparedStmt1 = database.getconn().prepareStatement("INSERT INTO REFERENCESS(CVE_ID,REF_TYPE,SOURCE,LINK)" + "VALUES ( ?, ?, ?, ?)");
+ PreparedStatement preparedStmt2 = database.getconn().prepareStatement("INSERT INTO SOFTWARES(CVE_ID,NAME)" + "VALUES ( ?, ?)");
+
+ System.out.println("Inserting " + list_id.size() + " data into VULNERABILITIES table ...");
+
for (int i = 0; i < list_id.size(); i++) {
- PreparedStatement preparedStmt = database.getconn().prepareStatement("INSERT INTO VULNERABILITIES(CVE_ID,PUB_DATE,MOD_DATE,SCORE,ACCESS_VECTOR,ACCESS_COMPLEXITY,AUTHENTICATION,CONFIDENTIALITY_IMPACT,INTEGRITY_IMPACT,AVAILABILITY_IMPACT,GEN_DATE,CWE_ID,SUMMARY)" + "VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? , ?, ?)");
+
preparedStmt.setString(1, list_id.get(i));
preparedStmt.setString(2, list_pub_date.get(i));
preparedStmt.setString(3, list_mod_date.get(i));
@@ -303,25 +324,32 @@ public class File_management {
preparedStmt.setString(13, list_sum.get(i));
preparedStmt.executeUpdate();
}
-
+
+ System.out.println("Inserting " + list_ref_type.size() + " data into REFERENCESS table ...");
+
for (int d = 0; d < list_ref_type.size(); d++) {
- PreparedStatement preparedStmt1 = database.getconn().prepareStatement("INSERT INTO REFERENCESS(CVE_ID,REF_TYPE,SOURCE,LINK)" + "VALUES ( ?, ?, ?, ?)");
+
preparedStmt1.setString(1, list_ref_cve.get(d));
preparedStmt1.setString(2, list_ref_type.get(d));
preparedStmt1.setString(3, list_ref_source.get(d));
preparedStmt1.setString(4, list_ref_link.get(d));
preparedStmt1.executeUpdate();
-
}
+ System.out.println("Inserting " + list_soft_name.size() + " data into SOFTWARES table ...");
+
for (int f = 0; f < list_soft_name.size(); f++) {
- PreparedStatement preparedStmt2 = database.getconn().prepareStatement("INSERT INTO SOFTWARES(CVE_ID,NAME)" + "VALUES ( ?, ?)");
+
preparedStmt2.setString(1, list_soft_cve.get(f));
preparedStmt2.setString(2, list_soft_name.get(f));
preparedStmt2.executeUpdate();
}
- System.out.println("Records inserted in the database: " + list_id.size());
+ preparedStmt.close();
+ preparedStmt1.close();
+ preparedStmt2.close();
+
+ System.out.println("Number of vulnerabilities inserted in the database: " + list_id.size());
System.out.println();
database.setTotalRecordsInDatabase(database.getTotalRecordsInDatabase() + list_id.size());
} catch (ParserConfigurationException | SAXException | IOException | DOMException | SQLException | NumberFormatException e) {
diff --git a/crawler/src/web/crawler/WebCrawler.java b/crawler/src/web/crawler/WebCrawler.java
index 08e5ce50dff7c874e6f2c015230d22a93a2fbdb6..2c682c796dd9bdb86b4a9f0e30664a6f75c3cbfd 100644
--- a/crawler/src/web/crawler/WebCrawler.java
+++ b/crawler/src/web/crawler/WebCrawler.java
@@ -34,9 +34,10 @@ public class WebCrawler {
public static void main(String[] args) throws ClassNotFoundException, InstantiationException, IllegalAccessException, SQLException, IOException, AWTException, Exception {
String FileNames[] = {
+ "nvdcve-2.0-2012.xml",
"nvdcve-2.0-2013.xml",
- "nvdcve-2.0-2014.xml",
- "nvdcve-2.0-2015.xml"
+ "nvdcve-2.0-2014.xml"
+ // "nvdcve-2.0-2015.xml"
};
//Database_creation database = new Database_creation();
database = new Database_creation();