initial commit of project

clean_dataset/combine_scraped_dataset_parts.py

@@ -0,0 +1,18 @@
+import json
+
+
+def main():
+    result_path = "data/dataset_fin.json"
+    dataset_start = 1
+    dataset_end = 18 # TODO: ADAPT to your number of dataset parts!!!
+    dataset_name = "data/dataset_parts/dataset_"
+    whole_dataset = {}
+    for curr_dataset_nr in range(dataset_start, dataset_end):
+        print("current dataset: " + str(curr_dataset_nr))
+        with open(dataset_name + str(curr_dataset_nr) + ".json", "r") as curr_json_file:
+            curr_dataset = json.load(curr_json_file)
+            whole_dataset.update(curr_dataset)
+    with open(result_path, "w") as output_file:
+        json.dump(whole_dataset, output_file)
+
+main()