From dcb9a22a0cf19daa33ab6e0d8b19b135730a8347 Mon Sep 17 00:00:00 2001 From: Jillian Anderson Date: Thu, 7 Jul 2016 14:17:56 -0400 Subject: [PATCH 1/3] Added new get_log functions trying to improve efficiency --- gitnet/get_log.py | 146 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 146 insertions(+) diff --git a/gitnet/get_log.py b/gitnet/get_log.py index 25ad71c..564ea13 100644 --- a/gitnet/get_log.py +++ b/gitnet/get_log.py @@ -244,3 +244,149 @@ def get_log(path, mode="stat", commit_source="local git"): source=commit_source, path=path, key_type=detect_key) + + +def new_get_log(path, mode="stat", commit_source="local git"): + if commit_source == "local git": + detect_key = "hash" + else: + detect_key = "unknown" + return CommitLog(dofd=new_helper(path, mode), + source=commit_source, + path=path, + key_type=detect_key) + + +def new_helper(path, mode="stat"): + print("Attempting local git log retrieval...") + # Log command modes, referenced by "mode" input. + log_commands = {"basic": ["git", "log"], "raw": ["git", "log", "--raw"], "stat": ["git", "log", "--stat"]} + if mode not in log_commands.keys(): + raise InputError("{} is not a valid retrieval mode.".format(mode)) + + # Save the current directory. Navigate to new directory. Retrieve logs. Return to original directory. + work_dir = os.getcwd() + os.chdir(path) + proc = sub.Popen(log_commands[mode], stdout=sub.PIPE) + + # Iterate through the lines and place them in the dictionary + collection = {} + sha = "" + for line in (iter(proc.stdout.readline, b'')): + string = line.decode('utf-8') + + # Identify and parse the string + if string != "\n": + id = new_identify(string) + # Commit Hash? + if id == "hash": + sha = string[7:14] + collection[sha] = {} + collection[sha]["hash"] = string[7:] + collection[sha]["mode"] = mode + # Author? + elif id == "author": + collection[sha]["author"] = string.split("<")[0][8:-1] + collection[sha]["email"] = string.split("<")[1][:-1] + # Date? + elif id == "date": + collection[sha]["date"] = string[8:] + # Message? + elif id == "message": + if "message" in collection[sha].keys(): + collection[sha]["message"] += " " + string[4:] + else: + collection[sha]["message"] = string[4:] + # File change record? + elif id == "change": + if "changes" in collection[sha].keys(): + collection[sha]["changes"].append(string[1:]) + collection[sha]["files"].append(string.split("|")[0].replace(" ", "")) + else: + collection[sha]["changes"] = [string[1:]] + collection[sha]["files"] = [string.split("|")[0].replace(" ", "")] + elif id == "summary": + collection[sha]["summary"] = string[1:] + # Filter numbers + temp = string.split(",") + for s in temp: + num = int("".join(list(filter(str.isdigit, s)))) + if "file" in s and "change" in s: + collection[sha]["fedits"] = num + if "insert" in s: + collection[sha]["inserts"] = num + if "delet" in s: + collection[sha]["deletes"] = num + elif id == "merge": + collection[sha]["merge"] = string[6:] + elif id == "multiple" or id == "none": + if "errors" in collection[sha].keys(): + collection[sha]["errors"].append(string) + else: + collection[sha]["errors"] = [string] + else: + warnings.warn("Parser was unable to identify {}. Identity string <{}> not recognized".format(line, id)) + os.chdir(work_dir) + + # If the retrieval was unsuccessful, raise an error. + if len(collection) == 0: + print("Raising error.") + if "true" in str(sub.Popen("git rev-parse --is-inside-work-tree", stdout=sub.PIPE).stdout): + raise RepositoryError("{} is not a Git repository.".format(path)) + else: + raise RepositoryError("{} has no commits.".format(path)) + # If the retrieval was successful, print a summary." + print("Got {} log records from: {}".format(len(collection), path)) + + return collection + + +def new_identify(s): + """ + A helper function for `parse_commits()`. It takes a string and attempts to identify it as an entry + field from a Git commit log. + + **Parameters** : + + > *s* : `string` + + >> One line of standard git log output (in basic, raw, or stat mode). + + **Return** : + + > A string identifying the type of data received. + + **Examples** : + + > `identify("commit 5be676481b4051af62f21eb2c8601b3f6bafb195") => "hash"` + > `identify("Author: JBWBecker ") => "author"` + > `identify("__init__.py | 2 ++") => "change"` + + """ + # identify checks whether the string matches an expected format. All matches are saved in a list. + matches = [] + if s[:6] == "commit" and len(s) == 48: + matches.append("hash") + if s[:7] == "Author:": + matches.append("author") + if s[:5] == "Date:": + matches.append("date") + if s[:4] == " ": + matches.append("message") + if (s[0] == " " and s[:4] != " " and "|" in s) or (s[0] == ":" and type(int(s[1:8])) == int): + matches.append("change") + if s[0] == " " and s[:4] != " " and (("insertion" in s and "(+)" in s) or ("deletion" in s and "(-)" in s)): + matches.append("summary") + if s[:6] == "Merge:": + matches.append("merge") + # If only one match was found, produce that string. Otherwise, produce "other" and raise a Warning. + if len(matches) == 1: + return matches[0] + elif len(matches) > 1: + warnings.warn("Unexpected parsing behaviour. <{}> matched multiple input patterns ({}) during parsing," + " so was identified as 'other'.".format(s,matches)) + return "multiple" + else: + warnings.warn("Unexpected parsing behaviour. <{}> did not match any input patterns during parsing," + " so was identified as 'other'.".format(s)) + return "none" \ No newline at end of file From 291ff2f60d994beb4b0081a80ce02e7b0cd26313 Mon Sep 17 00:00:00 2001 From: Jillian Anderson Date: Thu, 7 Jul 2016 16:34:16 -0400 Subject: [PATCH 2/3] Implemented generators rather than iterators in new get_log functions Increased memory efficiency substantially --- gitnet/__init__.py | 2 +- gitnet/get_log.py | 16 ++++++++++++++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/gitnet/__init__.py b/gitnet/__init__.py index 28bb8a1..a37bd1d 100644 --- a/gitnet/__init__.py +++ b/gitnet/__init__.py @@ -14,7 +14,7 @@ # If not, see . # ********************************************************************************************* -from .get_log import get_log +from .get_log import get_log, new_get_log from .exceptions import RepositoryError, ParseError, InputError from .log import Log from .commit_log import CommitLog diff --git a/gitnet/get_log.py b/gitnet/get_log.py index 564ea13..c33c0f7 100644 --- a/gitnet/get_log.py +++ b/gitnet/get_log.py @@ -272,7 +272,7 @@ def new_helper(path, mode="stat"): # Iterate through the lines and place them in the dictionary collection = {} sha = "" - for line in (iter(proc.stdout.readline, b'')): + for line in generate(proc.stdout): string = line.decode('utf-8') # Identify and parse the string @@ -389,4 +389,16 @@ def new_identify(s): else: warnings.warn("Unexpected parsing behaviour. <{}> did not match any input patterns during parsing," " so was identified as 'other'.".format(s)) - return "none" \ No newline at end of file + return "none" + + +def generate(stdout): + while True: + line = stdout.readline() + length = len(line) + if length > 1: + yield line + elif length == 1: + pass + else: + break From 50203e5f3906fce45f4a5af9f77b548f528c8521 Mon Sep 17 00:00:00 2001 From: Jillian Anderson Date: Fri, 8 Jul 2016 17:10:15 -0400 Subject: [PATCH 3/3] Implemented a third set of get_log functions. Minor improvements by rearranging if statements and using elif statements in the identify function --- gitnet/__init__.py | 2 +- gitnet/get_log.py | 179 +++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 172 insertions(+), 9 deletions(-) diff --git a/gitnet/__init__.py b/gitnet/__init__.py index a37bd1d..6f69f6a 100644 --- a/gitnet/__init__.py +++ b/gitnet/__init__.py @@ -14,7 +14,7 @@ # If not, see . # ********************************************************************************************* -from .get_log import get_log, new_get_log +from .get_log import get_log, new_get_log, get_log_v3 from .exceptions import RepositoryError, ParseError, InputError from .log import Log from .commit_log import CommitLog diff --git a/gitnet/get_log.py b/gitnet/get_log.py index c33c0f7..b2f36ee 100644 --- a/gitnet/get_log.py +++ b/gitnet/get_log.py @@ -20,6 +20,7 @@ from gitnet.exceptions import RepositoryError, ParseError, InputError from gitnet.commit_log import CommitLog import subprocess as sub +import line_profiler def retrieve_commits(path, mode="stat"): @@ -214,7 +215,6 @@ def parse_commits(commit_str): warnings.warn("Parser was unable to identify {}. Identity string <{}> not recognized".format(line,id)) return collection - def get_log(path, mode="stat", commit_source="local git"): """ A function for gathering data from a local Git repository. @@ -246,6 +246,7 @@ def get_log(path, mode="stat", commit_source="local git"): key_type=detect_key) +# New implementations def new_get_log(path, mode="stat", commit_source="local git"): if commit_source == "local git": detect_key = "hash" @@ -393,12 +394,174 @@ def new_identify(s): def generate(stdout): - while True: - line = stdout.readline() - length = len(line) - if length > 1: + line = stdout.readline() + while line != b"": + if len(line) > 1: yield line - elif length == 1: - pass + line = stdout.readline() + + +# Version 3 +def get_log_v3(path, mode="stat", commit_source="local git"): + if commit_source == "local git": + detect_key = "hash" + else: + detect_key = "unknown" + return CommitLog(dofd=helper_v3(path, mode), + source=commit_source, + path=path, + key_type=detect_key) + + +def helper_v3(path, mode="stat"): + print("Attempting local git log retrieval...") + # Log command modes, referenced by "mode" input. + log_commands = {"basic": ["git", "log"], "raw": ["git", "log", "--raw"], "stat": ["git", "log", "--stat"]} + if mode not in log_commands.keys(): + raise InputError("{} is not a valid retrieval mode.".format(mode)) + + # Save the current directory. Navigate to new directory. Retrieve logs. Return to original directory. + work_dir = os.getcwd() + os.chdir(path) + proc = sub.Popen(log_commands[mode], stdout=sub.PIPE) + stdout = generate(proc.stdout) + + # Iterate through the lines and place them in the dictionary + collection = {} + sha = "" + for line in stdout: + string = line.decode('utf-8') + + # Identify and parse the string + if string != "\n": + id = identify_v3(string) + # Commit Hash? + if id == "hash": + sha = string[7:14] + collection[sha] = {} + collection[sha]["hash"] = string[7:] + collection[sha]["mode"] = mode + + # File change record? + elif id == "change": + if "changes" in collection[sha].keys(): + collection[sha]["changes"].append(string[1:]) + collection[sha]["files"].append(string.split("|")[0].replace(" ", "")) + else: + collection[sha]["changes"] = [string[1:]] + collection[sha]["files"] = [string.split("|")[0].replace(" ", "")] + + # Message? + elif id == "message": + if "message" in collection[sha].keys(): + collection[sha]["message"] += " " + string[4:] + else: + collection[sha]["message"] = string[4:] + + # Author? + elif id == "author": + collection[sha]["author"] = string.split("<")[0][8:-1] + collection[sha]["email"] = string.split("<")[1][:-1] + + # Date? + elif id == "date": + collection[sha]["date"] = string[8:] + + # Summary? + elif id == "summary": + collection[sha]["summary"] = string[1:] + # Filter numbers + temp = string.split(",") + for s in temp: + num = int("".join(list(filter(str.isdigit, s)))) + if "file" in s and "change" in s: + collection[sha]["fedits"] = num + if "insert" in s: + collection[sha]["inserts"] = num + if "delet" in s: + collection[sha]["deletes"] = num + + # Merge? + elif id == "merge": + collection[sha]["merge"] = string[6:] + + # Error? + elif id == "multiple" or id == "none": + if "errors" in collection[sha].keys(): + collection[sha]["errors"].append(string) + else: + collection[sha]["errors"] = [string] + else: + warnings.warn("Parser was unable to identify {}. Identity string <{}> not recognized".format(line, id)) + os.chdir(work_dir) + + # If the retrieval was unsuccessful, raise an error. + if len(collection) == 0: + print("Raising error.") + if "true" in str(sub.Popen("git rev-parse --is-inside-work-tree", stdout=sub.PIPE).stdout): + raise RepositoryError("{} is not a Git repository.".format(path)) else: - break + raise RepositoryError("{} has no commits.".format(path)) + # If the retrieval was successful, print a summary." + print("Got {} log records from: {}".format(len(collection), path)) + + return collection + + +def identify_v3(s): + """ + A helper function for `parse_commits()`. It takes a string and attempts to identify it as an entry + field from a Git commit log. + + **Parameters** : + + > *s* : `string` + + >> One line of standard git log output (in basic, raw, or stat mode). + + **Return** : + + > A string identifying the type of data received. + + **Examples** : + + > `identify("commit 5be676481b4051af62f21eb2c8601b3f6bafb195") => "hash"` + > `identify("Author: JBWBecker ") => "author"` + > `identify("__init__.py | 2 ++") => "change"` + + """ + # identify checks whether the string matches an expected format. All matches are saved in a list. + matches = [] + + # Change? + if (s[0] == " " and s[:4] != " " and "|" in s) or (s[0] == ":" and type(int(s[1:8])) == int): + matches.append("change") + # Message? + elif s[:4] == " ": + matches.append("message") + # Hash? + elif s[:6] == "commit" and len(s) == 48: + matches.append("hash") + # Author? + elif s[:7] == "Author:": + matches.append("author") + # Date? + elif s[:5] == "Date:": + matches.append("date") + # Summary? + elif s[0] == " " and s[:4] != " " and (("insertion" in s and "(+)" in s) or ("deletion" in s and "(-)" in s)): + matches.append("summary") + # Merge? + elif s[:6] == "Merge:": + matches.append("merge") + # If only one match was found, produce that string. Otherwise, produce "other" and raise a Warning. + if len(matches) == 1: + return matches[0] + elif len(matches) > 1: + warnings.warn("Unexpected parsing behaviour. <{}> matched multiple input patterns ({}) during parsing," + " so was identified as 'other'.".format(s,matches)) + return "multiple" + else: + warnings.warn("Unexpected parsing behaviour. <{}> did not match any input patterns during parsing," + " so was identified as 'other'.".format(s)) + return "none" \ No newline at end of file