milos/uc_salary_aggregator
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

add medical_center flag: clinical/HCOMP faculty and medical-center clinical staff

Browse source 
Orthogonal boolean column alongside category, computed in classifier.py's
new is_medical_center(). Always flags MED CTR-tagged titles and the
existing HS CLIN PROF/-HCOMP patterns regardless of campus; additionally
flags generic clinical-workforce titles (physicians, nurses, dentists,
pharmacists, therapists, techs, etc.) only at the five campuses that
operate a UC-owned academic medical center (UCSF, UCLA, UCD, UCI, UCSD),
since those titles are indistinguishable from Student Health Services
staff elsewhere. SHS-tagged titles are always excluded.

Threaded through as a --group-by dimension for --aggregate-output and
as a column on --top-earners-output rows, alongside the per-employee CSV.
This commit is contained in:
Miloš Jovanović 2026-08-04 22:52:58 +00:00
parent efce6f3dec
commit 99d5108d76
2 changed files with 95 additions and 21 deletions
Download patch file Download diff file Expand all files Collapse all files

50
uc_salary_pipeline.py
View file

@ -33,18 +33,23 @@ expects, and writes the result.
OUTPUT
------
Per-employee CSV (--output), one row per employee per year:
year, campus, title, category, category_label,
year, campus, medical_center, title, category, category_label,
gross_pay, regular_pay, overtime_pay, other_pay, first_name, last_name
`medical_center` (true/false) marks employees working at a UC academic
medical center -- clinical/HCOMP faculty, physicians, nurses, and the
broader hospital workforce -- independent of `category`. See classifier.py's
is_medical_center() docstring for how it's determined.
Optionally, an aggregate CSV (--aggregate-output) with one row per
category (broken down further by --group-by year/campus):
[year] [campus] category, category_label,
category (broken down further by --group-by year/campus/medical_center):
[year] [campus] [medical_center] category, category_label,
employee_count, total_gross_pay, mean_gross_pay, median_gross_pay
Optionally, a top-earners CSV (--top-earners-output) with the --top-n
(default 10) highest gross_pay employees per category (also broken down
further by --group-by, e.g. per year):
category, category_label, rank, year, campus, title,
category, category_label, rank, year, campus, medical_center, title,
gross_pay, regular_pay, overtime_pay, other_pay, first_name, last_name
"""
@ -76,11 +81,13 @@ logging.basicConfig(
log = logging.getLogger(__name__)
OUTPUT_COLUMNS = [
"year", "campus", "title", "category", "category_label",
"year", "campus", "medical_center", "title", "category", "category_label",
"gross_pay", "regular_pay", "overtime_pay", "other_pay",
"first_name", "last_name",
]
GROUP_BY_DIMS = ("year", "campus", "medical_center")
def _dollar(s) -> float:
try:
@ -140,10 +147,6 @@ def process_file(path: Path, campus: str, year: int, writer: csv.writer,
with open(path, encoding="utf-8") as fh:
data = json.load(fh)
group_prefix = tuple(
year if dim == "year" else campus for dim in group_dims
)
count = 0
for record in data.get("records", []):
parsed = parse_uc_wages_record(record)
@ -155,21 +158,26 @@ def process_file(path: Path, campus: str, year: int, writer: csv.writer,
senior_threshold=senior_threshold,
support_threshold=support_threshold,
)
medical_center = classifier.is_medical_center(parsed["title"], campus)
writer.writerow([
year, campus, parsed["title"], category,
year, campus, medical_center, parsed["title"], category,
labels[category],
round(parsed["gross_pay"], 2), round(parsed["regular_pay"], 2),
round(parsed["overtime_pay"], 2), round(parsed["other_pay"], 2),
parsed["first_name"], parsed["last_name"],
])
count += 1
group_values = {"year": year, "campus": campus, "medical_center": medical_center}
group_prefix = tuple(group_values[dim] for dim in group_dims)
if agg is not None:
agg.setdefault(group_prefix + (category,), []).append(parsed["gross_pay"])
if console_stats is not None:
console_stats.setdefault((year, category), []).append(parsed["gross_pay"])
if top_earners is not None:
entry = (parsed["gross_pay"], next(tie_breaker),
(year, campus, parsed["title"], parsed["gross_pay"],
(year, campus, medical_center, parsed["title"], parsed["gross_pay"],
parsed["regular_pay"], parsed["overtime_pay"],
parsed["other_pay"], parsed["first_name"], parsed["last_name"]))
heap = top_earners.setdefault(group_prefix + (category,), [])
@ -206,8 +214,8 @@ def write_aggregate_csv(agg: dict, group_dims: tuple, labels: dict, outpath: Pat
def write_top_earners_csv(top_earners: dict, labels: dict, outpath: Path) -> None:
"""Write the top-N earners per (group_dims..., category) key, ranked
highest gross_pay first within each group."""
header = ["category", "category_label", "rank", "year", "campus", "title",
"gross_pay", "regular_pay", "overtime_pay", "other_pay",
header = ["category", "category_label", "rank", "year", "campus", "medical_center",
"title", "gross_pay", "regular_pay", "overtime_pay", "other_pay",
"first_name", "last_name"]
with open(outpath, "w", newline="", encoding="utf-8") as fh:
@ -217,10 +225,10 @@ def write_top_earners_csv(top_earners: dict, labels: dict, outpath: Path) -> Non
category = key[-1]
ranked = sorted(top_earners[key], key=lambda e: e[0], reverse=True)
for rank, (gross_pay, _, row) in enumerate(ranked, start=1):
year, campus, title, gross, regular, overtime, other, first, last = row
year, campus, medical_center, title, gross, regular, overtime, other, first, last = row
writer.writerow([
category, labels[category], rank,
year, campus, title, round(gross, 2), round(regular, 2),
year, campus, medical_center, title, round(gross, 2), round(regular, 2),
round(overtime, 2), round(other, 2), first, last,
])
log.info("Top earners CSV -> %s", outpath)
@ -293,14 +301,14 @@ def main():
help="If given, also write an aggregate CSV here with "
"employee count, total/mean/median gross pay per "
"category (see --group-by).")
parser.add_argument("--group-by", nargs="*", choices=["year", "campus"],
parser.add_argument("--group-by", nargs="*", choices=list(GROUP_BY_DIMS),
default=["year"],
help="Additional breakdown dimensions for "
"--aggregate-output and --top-earners-output, "
"alongside category (default: year). Pass both "
"for a year x campus x category breakdown, or "
"nothing to collapse everything to one group "
"per category.")
"alongside category (default: year). E.g. "
"--group-by year campus medical_center for the "
"finest breakdown, or nothing to collapse "
"everything to one group per category.")
parser.add_argument("--top-earners-output",
help="If given, also write a CSV of the --top-n "
"highest gross_pay employees per category (see "
@ -332,7 +340,7 @@ def main():
log.info("--skip-download set, classifying cached data only.")
labels = classifier.category_labels(args.senior_threshold, args.support_threshold)
group_dims = tuple(dim for dim in ("year", "campus") if dim in args.group_by)
group_dims = tuple(dim for dim in GROUP_BY_DIMS if dim in args.group_by)
agg = {} if args.aggregate_output else None
top_earners = {} if args.top_earners_output else None
tie_breaker = itertools.count() if top_earners is not None else None