1 files changed, 104 insertions, 0 deletions
diff --git a/main.py b/main.py
new file mode 100755
index 0000000..d7938f5
--- /dev/null
+++ b/main.py
@@ -0,0 +1,104 @@
+#!/usr/bin/env python3
+
+from bs4 import BeautifulSoup
+from multiprocessing import Pool
+import ntpath
+import os
+import csv
+from filing import Filing
+
+# Set directories
+root_dir = os.path.dirname(os.path.abspath(__file__))
+filings_dir = os.path.join(root_dir, "filings")
+clean_filings_dir = os.path.join(root_dir, "cleanedfilings")
+print("root_dir:", root_dir)
+print("filings_dir:", filings_dir)
+print("clean_filings_dir:", clean_filings_dir)
+
+# Read CSV data into a list of dicts
+csv_filename = os.path.join(root_dir, "data", "progress.csv")
+print("Reading CSV data from ", csv_filename)
+csv_data_rows = []
+with open(csv_filename, newline="") as csvfile:
+    try:
+        companyReader = csv.reader(csvfile, delimiter=";")
+        for row in companyReader:
+            csv_data_rows.append({
+                "ticker": row[0],
+                "cik": row[1],
+            })
+    except Exception as ex:
+        print("Reading CSV failed!")
+        raise ex
+
+# Construct a list of data_objects (dicts) that contain all necessary info
+# for processing.
+data_object_lst = []
+print("Writing filings to disk and building data objects")
+for row in csv_data_rows:
+    ticker = row["ticker"]
+    cik = row["cik"]
+
+    filings = Filing(cik=cik, filing_type="10-k", count=45)
+    filings.save(filings_dir)
+
+    input_file_dir = os.path.join(filings_dir, cik, "10-k")
+
+    input_filenames = []
+    for r, d, f in os.walk(input_file_dir):
+        for file_name in f:
+            input_filenames.append(os.path.join(r, file_name))
+
+    for input_filename in input_filenames:
+        output_filename = os.path.join(
+            clean_filings_dir, ticker, ntpath.basename(input_filename))
+
+        data_object = {
+            "ticker": ticker,
+            "cik": cik,
+            "input_filename": input_filename,
+            "output_filename": output_filename,
+            "input_string": None,
+            "output_string": None,
+        }
+        data_object_lst.append(data_object)
+
+# Read files and store content in data objects
+print("Reading filing data into memory")
+for data_object in data_object_lst:
+    input_filename = data_object["input_filename"]
+    with open(input_filename, "r+") as rawFile:
+        input_file_string = rawFile.read()
+        data_object["input_string"] = input_file_string
+
+# Multiprocessing worker function
+# Stores result in data objects
+
+
+def process(data_object):
+    input_string = data_object["input_string"]
+    output_string = BeautifulSoup(input_file_string, "lxml").text
+    data_object["output_string"] = output_string
+    return data_object
+
+
+print("Parsing filing data with lxml")
+
+# Multiprocessing, 4 cores
+with Pool(4) as pool:
+    result_data_objects = pool.map(process, data_object_lst)
+
+# No multiprocessing, does the same thing
+#result_data_objects = map(process, data_object_lst)
+
+# Write output to disk from data objects
+print("Writing filing data to disk")
+for data_object in result_data_objects:
+    output_filename = data_object["output_filename"]
+    output_dir = os.path.dirname(output_filename)
+    os.makedirs(output_dir, exist_ok=True)
+    with open(output_filename, "w") as newFile:
+        output_string = data_object["output_string"]
+        newFile.write(output_string)
+
+print("Done.")