diff options
| author | Jan Tuomi <jan.tuomi@eficode.com> | 2019-10-09 09:29:11 +0300 |
|---|---|---|
| committer | Jan Tuomi <jan.tuomi@eficode.com> | 2019-10-09 09:29:11 +0300 |
| commit | ae1033142801c1a0410d878a7f798c6aad217640 (patch) | |
| tree | 4228678f1b09f565c6445cb5a1ac5fc7449181ba | |
Initial commit
| -rw-r--r-- | .gitignore | 5 | ||||
| -rw-r--r-- | data/progress.csv | 3 | ||||
| -rw-r--r-- | filing.py | 20 | ||||
| -rwxr-xr-x | main.py | 104 | ||||
| -rw-r--r-- | ntpath.py | 7 |
5 files changed, 139 insertions, 0 deletions
diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c5b4e54 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +venv/ +filings/ +cleanedfilings/ +__pycache__ +.vscode/ diff --git a/data/progress.csv b/data/progress.csv new file mode 100644 index 0000000..fc5c3a2 --- /dev/null +++ b/data/progress.csv @@ -0,0 +1,3 @@ +JNJ;0000200406 +NIO;0000123123 +DPZ;00000456456 diff --git a/filing.py b/filing.py new file mode 100644 index 0000000..7d90507 --- /dev/null +++ b/filing.py @@ -0,0 +1,20 @@ +import os +uid = 1 + +# Placeholder Filing class + + +class Filing: + def __init__(self, cik, filing_type, count): + self.cik = cik + self.filing_type = filing_type + self.count = count + + def save(self, filings_dir): + global uid + out_dir = os.path.join(filings_dir, self.cik, "10-k") + out_filename = os.path.join(out_dir, "file_{}.out".format(uid)) + uid += 1 + os.makedirs(out_dir, exist_ok=True) + with open(out_filename, "w") as f: + f.write("<html>ribul</html>") @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 + +from bs4 import BeautifulSoup +from multiprocessing import Pool +import ntpath +import os +import csv +from filing import Filing + +# Set directories +root_dir = os.path.dirname(os.path.abspath(__file__)) +filings_dir = os.path.join(root_dir, "filings") +clean_filings_dir = os.path.join(root_dir, "cleanedfilings") +print("root_dir:", root_dir) +print("filings_dir:", filings_dir) +print("clean_filings_dir:", clean_filings_dir) + +# Read CSV data into a list of dicts +csv_filename = os.path.join(root_dir, "data", "progress.csv") +print("Reading CSV data from ", csv_filename) +csv_data_rows = [] +with open(csv_filename, newline="") as csvfile: + try: + companyReader = csv.reader(csvfile, delimiter=";") + for row in companyReader: + csv_data_rows.append({ + "ticker": row[0], + "cik": row[1], + }) + except Exception as ex: + print("Reading CSV failed!") + raise ex + +# Construct a list of data_objects (dicts) that contain all necessary info +# for processing. +data_object_lst = [] +print("Writing filings to disk and building data objects") +for row in csv_data_rows: + ticker = row["ticker"] + cik = row["cik"] + + filings = Filing(cik=cik, filing_type="10-k", count=45) + filings.save(filings_dir) + + input_file_dir = os.path.join(filings_dir, cik, "10-k") + + input_filenames = [] + for r, d, f in os.walk(input_file_dir): + for file_name in f: + input_filenames.append(os.path.join(r, file_name)) + + for input_filename in input_filenames: + output_filename = os.path.join( + clean_filings_dir, ticker, ntpath.basename(input_filename)) + + data_object = { + "ticker": ticker, + "cik": cik, + "input_filename": input_filename, + "output_filename": output_filename, + "input_string": None, + "output_string": None, + } + data_object_lst.append(data_object) + +# Read files and store content in data objects +print("Reading filing data into memory") +for data_object in data_object_lst: + input_filename = data_object["input_filename"] + with open(input_filename, "r+") as rawFile: + input_file_string = rawFile.read() + data_object["input_string"] = input_file_string + +# Multiprocessing worker function +# Stores result in data objects + + +def process(data_object): + input_string = data_object["input_string"] + output_string = BeautifulSoup(input_file_string, "lxml").text + data_object["output_string"] = output_string + return data_object + + +print("Parsing filing data with lxml") + +# Multiprocessing, 4 cores +with Pool(4) as pool: + result_data_objects = pool.map(process, data_object_lst) + +# No multiprocessing, does the same thing +#result_data_objects = map(process, data_object_lst) + +# Write output to disk from data objects +print("Writing filing data to disk") +for data_object in result_data_objects: + output_filename = data_object["output_filename"] + output_dir = os.path.dirname(output_filename) + os.makedirs(output_dir, exist_ok=True) + with open(output_filename, "w") as newFile: + output_string = data_object["output_string"] + newFile.write(output_string) + +print("Done.") diff --git a/ntpath.py b/ntpath.py new file mode 100644 index 0000000..8fdfeb6 --- /dev/null +++ b/ntpath.py @@ -0,0 +1,7 @@ +import os + +# Kikka kakkonen because I don't have Windows + + +def basename(*args, **kwargs): + return os.path.basename(*args, **kwargs) |