config/config.yaml

@@ -26,7 +26,7 @@ log:
   console: true
 
 sources:
-  - path: /data/y4/tools/data/users-0.csv
+  - path: /data/y4/tools/data/user-0s.csv
     csv:
       delimiter: "|"
       withHeader: true
@@ -86,7 +86,7 @@ sources:
             type: "STRING"
             index: 16
 
-  - path: /data/y4/tools/data/posts-0.csv
+  - path: /data/y4/tools/data/post-0s.csv
     csv:
       delimiter: "|"
       withHeader: true
@@ -140,7 +140,7 @@ sources:
             type: "INT"
             index: 14
 
-  - path: /data/y4/tools/data/groups-0.csv
+  - path: /data/y4/tools/data/group-0s.csv
     csv:
       delimiter: "|"
       withHeader: true
@@ -161,7 +161,7 @@ sources:
             type: "STRING"
             index: 3
 
-  - path: /data/y4/tools/data/organizations-0.csv
+  - path: /data/y4/tools/data/organization-0s.csv
     csv:
       delimiter: "|"
       withHeader: true
@@ -185,7 +185,7 @@ sources:
             type: "STRING"
             index: 4
 
-  - path: /data/y4/tools/data/tasks-0.csv
+  - path: /data/y4/tools/data/task-0s.csv
     csv:
       delimiter: "|"
       withHeader: true
@@ -284,25 +284,25 @@ sources:
             type: "STRING"
             index: 0
 
-  - path: /data/y4/tools/data/post_repost_post-0.csv
+#  - path: /data/y4/tools/data/post_repost_post-0.csv
-    csv:
+#    csv:
-      delimiter: "|"
+#      delimiter: "|"
-      withHeader: true
+#      withHeader: true
-    edges:
+#    edges:
-      - name: post_repost_post
+#      - name: post_repost_post
-        mode: INSERT
+#        mode: INSERT
-        src:
+#        src:
-          id:
+#          id:
-            type: "STRING"
+#            type: "STRING"
-            index: 2 # src
+#            index: 2 # src
-        dst:
+#        dst:
-          id:
+#          id:
-            type: "STRING"
+#            type: "STRING"
-            index: 1 # dst
+#            index: 1 # dst
-        props:
+#        props:
-          - name: "extra_prop"
+#          - name: "extra_prop"
-            type: "STRING"
+#            type: "STRING"
-            index: 0
+#            index: 0
 
   - path: /data/y4/tools/data/post_cite_post-0.csv
     csv:
@@ -324,25 +324,25 @@ sources:
             type: "STRING"
             index: 0
 
-  - path: /data/y4/tools/data/user_post_post-0.csv
+#  - path: /data/y4/tools/data/user_post_post-0.csv
-    csv:
+#    csv:
-      delimiter: "|"
+#      delimiter: "|"
-      withHeader: true
+#      withHeader: true
-    edges:
+#    edges:
-      - name: user_post_post
+#      - name: user_post_post
-        mode: INSERT
+#        mode: INSERT
-        src:
+#        src:
-          id:
+#          id:
-            type: "STRING"
+#            type: "STRING"
-            index: 2 # src
+#            index: 2 # src
-        dst:
+#        dst:
-          id:
+#          id:
-            type: "STRING"
+#            type: "STRING"
-            index: 1 # dst
+#            index: 1 # dst
-        props:
+#        props:
-          - name: "extra_prop"
+#          - name: "extra_prop"
-            type: "STRING"
+#            type: "STRING"
-            index: 0
+#            index: 0
 
   - path: /data/y4/tools/data/user_comment_user-0.csv
     csv:
@@ -544,22 +544,22 @@ sources:
             type: "STRING"
             index: 0
 
-  - path: /data/y4/tools/data/user_join_organization-0.csv
+#  - path: /data/y4/tools/data/user_join_organization-0.csv
-    csv:
+#    csv:
-      delimiter: "|"
+#      delimiter: "|"
-      withHeader: true
+#      withHeader: true
-    edges:
+#    edges:
-      - name: user_join_organization
+#      - name: user_join_organization
-        mode: INSERT
+#        mode: INSERT
-        src:
+#        src:
-          id:
+#          id:
-            type: "STRING"
+#            type: "STRING"
-            index: 2 # src
+#            index: 2 # src
-        dst:
+#        dst:
-          id:
+#          id:
-            type: "STRING"
+#            type: "STRING"
-            index: 1 # dst
+#            index: 1 # dst
-        props:
+#        props:
-          - name: "extra_prop"
+#          - name: "extra_prop"
-            type: "STRING"
+#            type: "STRING"
-            index: 0
+#            index: 0

src/main/java/com/idata/tools/GeneratorTestData.java

@@ -341,23 +341,15 @@ public class GeneratorTestData {
         int i = 0;
         List<String> edgeLists = Arrays.asList("post_in_task", "user_participate_task", "user_join_group", "user_join_organization");
         for (String src : srcList) {
+            i++;
             if (i % 1000 == 0) {
                 log.info("edge_type: {}-{}, {}",edgeType, count, i);
             }
 
             List<String> dsts;
             if (i == 0 && !edgeLists.contains(edgeType)) {
-                // 当索引等于0 时，选择 1000 个目标节点
-                dsts = getRandomSample(dstList, 1000);
-                log.info("super vertex id is {}; dst size is {}", src, dsts.size());
-            } else if (i == 1 && !edgeLists.contains(edgeType)) {
-                // 当索引等于0 时，选择 10001 个目标节点
-                dsts = getRandomSample(dstList, 1500);
-                log.info("ssuper vertex id is {}; dst size is {}", src, dsts.size());
-            } else if (i == 2 && !edgeLists.contains(edgeType)) {
                 // 当索引等于0 时，选择 10001 个目标节点
                 dsts = getRandomSample(dstList, 10001);
-                log.info("sssuper vertex id is {}; dst size is {}", src, dsts.size());
             } else {
                 // 否则选择最多 num 个目标节点
                 dsts = getRandomSample(dstList, Math.min(dstList.size(), num));
@@ -372,7 +364,6 @@ public class GeneratorTestData {
                 edge.put("edge_type", edgeType);
                 edges.add(edge);
             }
-            i++;
         }
         return edges;
     }