Master

app/src/main/java/githubsqlfinder/App.java

@@ -36,11 +36,11 @@ public class App {
     static int numberOfAddedQueriesPostgreSQL = 0;
     static int numberOfAddedQueriesTSql = 0;
     static long timeMs = 0;
+    static int numberOfValidFiles = 0;
 
     public static void main(String[] args) throws Exception {
 
         long sTime = System.nanoTime();
-
         int numberOfGithubFilesToSearch = 100;
         int batchSize = 100;
         int mode = 2;
@@ -55,7 +55,6 @@ public class App {
                 System.err.println("The first parameter (number of files to process) was incorrect");
                 return;
             }
-           
         }
 
         if (args.length > 1) {
@@ -101,7 +100,7 @@ public class App {
         //
         System.out.println("Number of files to process: " + numberOfGithubFilesToSearch);
         System.out.println("Batch size: " + batchSize);
-        System.out.println("Mode (1 - Example files, 2 - Github, 3 - Clear DB): " + mode);
+        System.out.println("Mode (1 - Example files, 2 - Github, 3 - Clear DB, 4 - Show DB): " + mode);
         System.out.println("Data (1 - Sample data, 2 - Real data): " + sample);
         System.out.println("offset: " + offset);
         
@@ -109,7 +108,6 @@ public class App {
         boolean repeatCycle = true;
         String input = "";
 
-        
         if (mode == 1) {
             //example files
             int maxFile = 3;
@@ -120,14 +118,18 @@ public class App {
 
                 input = Files.readString(path);
 
-                runPhpAntlr(input, "example", filePath);
+                if (isValidFile(input)) {
+                    runPhpAntlr(input, "example", filePath);
 
-                for (String query : stringStatements) {
-                    if (isValidQuery(query)) {
-                        runAntlr(query.replaceAll("\\\\", ""), "example", filePath);
+                    for (String query : stringStatements) {
+                        if (isValidQuery(query)) {
+                            runAntlr(query.replaceAll("\\\\", ""), "example", filePath);
+                        }
                     }
                 }
             }
+        } else if (mode == 4) {
+            db.showPage(0, true);
         } else if (mode == 2) {
             //github
 
@@ -149,17 +151,27 @@ public class App {
                 githubTimeMs += (endTime - startTime) / 1_000_000;
 
                 for (FieldValueList row : result.iterateAll()) {
-
-                    input = row.get("content").getStringValue();
                     String repoName = row.get("repo_name").getStringValue();
                     String filePath = row.get("path").getStringValue();
+                    if (sample == 1) {
+                        input = row.get("content").getStringValue();
+                    } else {
+                        startTime = System.nanoTime();
+                        input = githubFinder.getFileContent(repoName, filePath);
+                        endTime = System.nanoTime();
+
+                        githubTimeMs += (endTime - startTime) / 1_000_000;
+                    }
 
-                    runPhpAntlr(input, repoName, filePath);
+                    if (isValidFile(input)) {
+                        numberOfValidFiles++;
+                        runPhpAntlr(input, repoName, filePath);
 
-                    for (String query : stringStatements) {
-                        if (isValidQuery(query)) {
-                            runAntlr(query.replaceAll("\\\\", ""), repoName, filePath);
-                        }                        
+                        for (String query : stringStatements) {
+                            if (isValidQuery(query)) {
+                                runAntlr(query.replaceAll("\\\\", ""), repoName, filePath);
+                            }                        
+                        }
                     }
                     index++;
                     System.out.println("" + index + "/" + numberOfGithubFilesToSearch);
@@ -182,11 +194,12 @@ public class App {
         timeMs += (eTime - sTime) / 1_000_000;
 
         if (mode == 2) {
-            System.out.println("Github requests time in ms: " + githubTimeMs + "(" + String.format("%.2f", ((double) githubTimeMs / timeMs) * 100) + "%)");
-            System.out.println("ANTLR parsing time in ms: " + antlrTimeMs + "(" + String.format("%.2f", ((double) antlrTimeMs / timeMs) * 100) + "%)");
-            System.out.println("Parsing tree string finding: " + parsingTreeTimeMs + "(" + String.format("%.2f", ((double) parsingTreeTimeMs / timeMs) * 100) + "%)");
+            System.out.println("Github requests time in ms: " + githubTimeMs + " (" + String.format("%.2f", ((double) githubTimeMs / timeMs) * 100) + "%)");
+            System.out.println("ANTLR parsing time in ms: " + antlrTimeMs + " (" + String.format("%.2f", ((double) antlrTimeMs / timeMs) * 100) + "%)");
+            System.out.println("Parsing tree string finding in ms: " + parsingTreeTimeMs + " (" + String.format("%.2f", ((double) parsingTreeTimeMs / timeMs) * 100) + "%)");
             System.out.println("Whole time: " + timeMs);
-            System.out.println("Number of all found queries: " + numberOfAddedQueries + "(from " + numberOfGithubFilesToSearch + " files)");
+            System.out.println("Number of valid files: " + numberOfValidFiles);
+            System.out.println("Number of all found queries: " + numberOfAddedQueries + " (from " + numberOfGithubFilesToSearch + " files)");
             System.out.println("Number of TSql queries: " + numberOfAddedQueriesTSql);
             System.out.println("Number of Postgre SQL queries: " + numberOfAddedQueriesPostgreSQL);
             System.out.println("Number of PlSql queries: " + numberOfAddedQueriesPlSql);
@@ -237,6 +250,7 @@ public class App {
                 query = "";
                 paramIndex = 0;
                 getParametrizedQuery(arithmeticExpressionContext);
+                query = removeQuotesAroundParams(query);
                 if (!query.equals("") && !stringStatements.contains(query)) {
                     stringStatements.add(query);
                 }
@@ -276,6 +290,25 @@ public class App {
         }
         matcher.appendTail(result);
 
+        return removeQuotesAroundParams(result.toString());
+    }
+
+    public static String removeQuotesAroundParams(String input) {
+        if (input == null) {
+            return null;
+        }
+
+        Pattern pattern = Pattern.compile("(['\"]?):p\\d+\\1");
+        Matcher matcher = pattern.matcher(input);
+        StringBuffer result = new StringBuffer();
+
+        while (matcher.find()) {
+            String match = matcher.group();
+            String replacement = match.replaceAll("['\"]", "");
+            matcher.appendReplacement(result, replacement);
+        }
+        matcher.appendTail(result);
+
         return result.toString();
     }
 
@@ -283,6 +316,7 @@ public class App {
         if (tree instanceof StringContext) {
             String text = tree.getText();
             text = text.replaceAll("\\\\", "");
+
             if (((text.startsWith("\'") && text.endsWith("\'")) || (text.startsWith("\"") && text.endsWith("\""))) && text.length() > 1) {
                 text = text.substring(1, text.length() - 1);
                 query += text;
@@ -318,6 +352,25 @@ public class App {
     
         return false;
     }
+
+    public static boolean isValidFile(String file) {
+        if (file == null) {
+            return false;
+        }
+        
+        String lowerCaseFile = file.toLowerCase();
+
+        if (lowerCaseFile.contains("select") && lowerCaseFile.contains("from") && lowerCaseFile.indexOf("select") < lowerCaseFile.indexOf("from")) {
+            return true;
+        }
+
+        if (lowerCaseFile.contains("create table")) {
+            return true;
+        }
+    
+        return false;
+    }
+
     public static void runAntlr(String input, String repoName, String filePath) {
         JDBC db = new JDBC();
         input = input.toUpperCase();