add formatted per-category summary table printed to stdout after each run
Break the console summary out by year instead of collapsing every requested year into one set of totals.
This commit is contained in:
parent
8d767e6523
commit
a621b94512
1 changed files with 55 additions and 2 deletions
|
|
@ -122,7 +122,8 @@ def process_file(path: Path, campus: str, year: int, writer: csv.writer,
|
||||||
senior_threshold: float, support_threshold: float,
|
senior_threshold: float, support_threshold: float,
|
||||||
agg: dict | None = None, group_dims: tuple = (),
|
agg: dict | None = None, group_dims: tuple = (),
|
||||||
top_earners: dict | None = None, top_n: int = 10,
|
top_earners: dict | None = None, top_n: int = 10,
|
||||||
tie_breaker: itertools.count | None = None) -> int:
|
tie_breaker: itertools.count | None = None,
|
||||||
|
console_stats: dict | None = None) -> int:
|
||||||
"""Classify every record in one UC-wages JSON file, writing a CSV row
|
"""Classify every record in one UC-wages JSON file, writing a CSV row
|
||||||
per employee. Returns the number of rows written.
|
per employee. Returns the number of rows written.
|
||||||
|
|
||||||
|
|
@ -131,6 +132,10 @@ def process_file(path: Path, campus: str, year: int, writer: csv.writer,
|
||||||
|
|
||||||
If `top_earners` is given, also maintains a size-`top_n` min-heap of
|
If `top_earners` is given, also maintains a size-`top_n` min-heap of
|
||||||
the highest gross_pay employees per (group_dims..., category) key.
|
the highest gross_pay employees per (group_dims..., category) key.
|
||||||
|
|
||||||
|
If `console_stats` is given, also accumulates each record's gross_pay
|
||||||
|
under its (year, category) for the end-of-run summary tables printed
|
||||||
|
to stdout (one table per year).
|
||||||
"""
|
"""
|
||||||
with open(path, encoding="utf-8") as fh:
|
with open(path, encoding="utf-8") as fh:
|
||||||
data = json.load(fh)
|
data = json.load(fh)
|
||||||
|
|
@ -160,6 +165,8 @@ def process_file(path: Path, campus: str, year: int, writer: csv.writer,
|
||||||
count += 1
|
count += 1
|
||||||
if agg is not None:
|
if agg is not None:
|
||||||
agg.setdefault(group_prefix + (category,), []).append(parsed["gross_pay"])
|
agg.setdefault(group_prefix + (category,), []).append(parsed["gross_pay"])
|
||||||
|
if console_stats is not None:
|
||||||
|
console_stats.setdefault((year, category), []).append(parsed["gross_pay"])
|
||||||
if top_earners is not None:
|
if top_earners is not None:
|
||||||
entry = (parsed["gross_pay"], next(tie_breaker),
|
entry = (parsed["gross_pay"], next(tie_breaker),
|
||||||
(year, campus, parsed["title"], parsed["gross_pay"],
|
(year, campus, parsed["title"], parsed["gross_pay"],
|
||||||
|
|
@ -219,6 +226,44 @@ def write_top_earners_csv(top_earners: dict, outpath: Path) -> None:
|
||||||
log.info("Top earners CSV -> %s", outpath)
|
log.info("Top earners CSV -> %s", outpath)
|
||||||
|
|
||||||
|
|
||||||
|
def _money(v: float) -> str:
|
||||||
|
if abs(v) >= 1_000_000:
|
||||||
|
return f"${v / 1_000_000:,.2f}M"
|
||||||
|
return f"${v:,.0f}"
|
||||||
|
|
||||||
|
|
||||||
|
def print_console_summary(console_stats: dict) -> None:
|
||||||
|
"""Print one formatted category-by-category summary table per year,
|
||||||
|
keyed off (year, category) -> [gross_pay, ...] entries in console_stats."""
|
||||||
|
label_w, col_w = 34, 16
|
||||||
|
header = (f"{'Category':<{label_w}}{'Employees':>{col_w}}"
|
||||||
|
f"{'Total Pay':>{col_w}}{'Median Pay':>{col_w}}")
|
||||||
|
rule = "=" * len(header)
|
||||||
|
|
||||||
|
years = sorted({year for year, _ in console_stats})
|
||||||
|
for year in years:
|
||||||
|
print(f"\n{rule}")
|
||||||
|
print(f"UC SALARY CLASSIFICATION SUMMARY -- {year}")
|
||||||
|
print(rule)
|
||||||
|
print(header)
|
||||||
|
print("-" * len(header))
|
||||||
|
|
||||||
|
total_count, total_pay = 0, 0.0
|
||||||
|
for category in classifier.CATEGORIES:
|
||||||
|
values = console_stats.get((year, category), [])
|
||||||
|
count, total = len(values), sum(values)
|
||||||
|
median = statistics.median(values) if values else 0.0
|
||||||
|
total_count += count
|
||||||
|
total_pay += total
|
||||||
|
label = classifier.CATEGORY_LABELS[category]
|
||||||
|
print(f"{label:<{label_w}}{count:>{col_w},}"
|
||||||
|
f"{_money(total):>{col_w}}{_money(median):>{col_w}}")
|
||||||
|
|
||||||
|
print("-" * len(header))
|
||||||
|
print(f"{'TOTAL':<{label_w}}{total_count:>{col_w},}{_money(total_pay):>{col_w}}")
|
||||||
|
print(rule)
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# CLI
|
# CLI
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
@ -266,6 +311,9 @@ def main():
|
||||||
parser.add_argument("--skip-download", action="store_true",
|
parser.add_argument("--skip-download", action="store_true",
|
||||||
help="Classify whatever is already cached in "
|
help="Classify whatever is already cached in "
|
||||||
"--data-dir without hitting the network.")
|
"--data-dir without hitting the network.")
|
||||||
|
parser.add_argument("--quiet", action="store_true",
|
||||||
|
help="Suppress the formatted summary table printed "
|
||||||
|
"to stdout after processing.")
|
||||||
parser.add_argument("--senior-threshold", type=float,
|
parser.add_argument("--senior-threshold", type=float,
|
||||||
default=classifier.SENIOR_STAFF_THRESHOLD,
|
default=classifier.SENIOR_STAFF_THRESHOLD,
|
||||||
help=f"Gross pay threshold for SENIOR_PROF_STAFF "
|
help=f"Gross pay threshold for SENIOR_PROF_STAFF "
|
||||||
|
|
@ -287,6 +335,7 @@ def main():
|
||||||
agg = {} if args.aggregate_output else None
|
agg = {} if args.aggregate_output else None
|
||||||
top_earners = {} if args.top_earners_output else None
|
top_earners = {} if args.top_earners_output else None
|
||||||
tie_breaker = itertools.count() if top_earners is not None else None
|
tie_breaker = itertools.count() if top_earners is not None else None
|
||||||
|
console_stats = {} if not args.quiet else None
|
||||||
|
|
||||||
output_path = Path(args.output)
|
output_path = Path(args.output)
|
||||||
total_rows = 0
|
total_rows = 0
|
||||||
|
|
@ -304,7 +353,8 @@ def main():
|
||||||
args.senior_threshold, args.support_threshold,
|
args.senior_threshold, args.support_threshold,
|
||||||
agg=agg, group_dims=group_dims,
|
agg=agg, group_dims=group_dims,
|
||||||
top_earners=top_earners, top_n=args.top_n,
|
top_earners=top_earners, top_n=args.top_n,
|
||||||
tie_breaker=tie_breaker)
|
tie_breaker=tie_breaker,
|
||||||
|
console_stats=console_stats)
|
||||||
total_rows += count
|
total_rows += count
|
||||||
log.info("%s %s: classified %d records", campus, year, count)
|
log.info("%s %s: classified %d records", campus, year, count)
|
||||||
|
|
||||||
|
|
@ -320,6 +370,9 @@ def main():
|
||||||
if top_earners is not None:
|
if top_earners is not None:
|
||||||
write_top_earners_csv(top_earners, Path(args.top_earners_output))
|
write_top_earners_csv(top_earners, Path(args.top_earners_output))
|
||||||
|
|
||||||
|
if console_stats is not None:
|
||||||
|
print_console_summary(console_stats)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
main()
|
main()
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue