initial commit
This commit is contained in:
Miloš Jovanović
commit
9fafcb649f
98 changed files with 3386328 additions and 0 deletions
916
uc_salary_pipeline.py
Normal file
916
uc_salary_pipeline.py
Normal file
|
|
@ -0,0 +1,916 @@
|
|||
"""
|
||||
uc_salary_pipeline.py
|
||||
=====================
|
||||
All-in-one pipeline: scrape UCOP Annual Wage data, classify every employee
|
||||
into one of 11 analytical categories, and produce a summary CSV.
|
||||
|
||||
USAGE
|
||||
-----
|
||||
# Download raw data AND classify:
|
||||
python uc_salary_pipeline.py --years 2015 2024 --scrape --output summary.csv
|
||||
|
||||
# Classify already-downloaded CSVs (skips scraping):
|
||||
python uc_salary_pipeline.py --files UCOP_Data_2015.csv UCOP_Data_2024.csv --output summary.csv
|
||||
|
||||
# Classify Transparent California CSVs instead:
|
||||
python uc_salary_pipeline.py --files tc_2015.csv tc_2024.csv --source tc --output summary.csv
|
||||
|
||||
DATA ACQUISITION
|
||||
----------------
|
||||
Scraping is delegated to the upstream tool:
|
||||
https://github.com/raleighlittles/UC_Employee_Salary_Data
|
||||
|
||||
On first run with --scrape, this script:
|
||||
1. Checks for ./UC_Employee_Salary_Data/.git
|
||||
2. If absent, git clones the repo (git must be on PATH)
|
||||
3. Runs uc_employee_salary_downloader.py -y <YEAR> for each year
|
||||
4. Uses the produced UCOP_Data_<YEAR>.csv files for classification
|
||||
|
||||
Subsequent runs reuse existing CSVs without re-scraping.
|
||||
Pass --files to skip acquisition entirely.
|
||||
|
||||
CATEGORIES (11 total)
|
||||
---------------------
|
||||
1. LADDER_FACULTY Senate professorial series (PROF-AY/FY/HCOMP, all ranks)
|
||||
+ Professor of Teaching / SOE/PSOE (Senate)
|
||||
+ AES tenure Agronomists, Astronomers
|
||||
2. HS_CLINICAL Health Sciences Compensation Plan clinical series:
|
||||
HS CLIN PROF, HS ASSOC CLIN PROF, HS ASST CLIN PROF
|
||||
(these are ladder-equivalent but on HCOMP; analytically
|
||||
distinct because of large "other pay" from clinical income)
|
||||
3. ACTING_PROFESSOR Acting ranks (both Senate ACT PROF and non-Senate ACT ASST PROF)
|
||||
and Visiting Professors / Recall appointments
|
||||
4. PROF_IN_RESIDENCE Professor in Residence series (CTO 311); campus-funded
|
||||
non-tenure research faculty
|
||||
5. OTHER_ACADEMIC Unit 18 lecturers, Adjunct Professors, Prof of Clinical
|
||||
(non-HS), research series (Proj Sci, Specialist, Coop Ext),
|
||||
Librarians, Academic Coordinators/Administrators, Postdocs
|
||||
6. SENIOR_MANAGEMENT Chancellor, Provost, Vice Chancellor, Vice President,
|
||||
Dean (all levels), Executive Director, Head Coach
|
||||
7. STUDENT_TRAINEE TAs, GSRs, Associates (GSHIP), Interns, Residents,
|
||||
Postdoctoral Scholars
|
||||
8. SENIOR_PROF_STAFF High-compensation non-academic staff (gross pay ≥ $100k):
|
||||
Physicians, Nurse Practitioners, Senior IT/Analysts,
|
||||
Senior Administrators, Engineers, Attorneys, Finance
|
||||
9. PROF_STAFF Mid-range professional/administrative staff ($40k–$100k)
|
||||
10. SUPPORT_STAFF Lower-wage operational staff (< $40k): food service,
|
||||
custodial, parking, entry-level clerical, security
|
||||
11. ATHLETICS Athletic coaches and athletics admin (separated from SMG
|
||||
because their "other pay" bonuses massively distort SMG totals)
|
||||
|
||||
Pay thresholds for SENIOR_PROF_STAFF vs PROF_STAFF vs SUPPORT_STAFF are
|
||||
applied to gross_pay and can be adjusted via --senior-threshold and
|
||||
--support-threshold flags.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ---------------------------------------------------------------------------
|
||||
# LOGGING
|
||||
# ---------------------------------------------------------------------------
|
||||
logging.basicConfig(
|
||||
format="%(asctime)s [%(levelname)s] %(message)s",
|
||||
level=logging.INFO,
|
||||
handlers=[
|
||||
logging.FileHandler("uc_salary_pipeline.log", encoding="utf-8"),
|
||||
logging.StreamHandler(sys.stdout),
|
||||
],
|
||||
)
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# UPSTREAM SCRAPER DEPENDENCY
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
SCRAPER_REPO = "https://github.com/raleighlittles/UC_Employee_Salary_Data"
|
||||
SCRAPER_DIR = Path("UC_Employee_Salary_Data")
|
||||
SCRAPER_SCRIPT = SCRAPER_DIR / "src" / "uc_employee_salary_downloader.py"
|
||||
UCOP_COLUMNS = ["id", "year", "location", "first name", "last name",
|
||||
"title", "gross pay", "regular pay", "overtime pay", "other pay"]
|
||||
|
||||
|
||||
def _patch_scraper() -> None:
|
||||
"""
|
||||
Apply fixes to the upstream scraper after cloning, in-place.
|
||||
Idempotent: a sentinel comment is checked before each run.
|
||||
|
||||
Fix 1 - stale hardcoded Cookie header (causes HTTP 400)
|
||||
Remove the baked-in JSESSIONID/AWSALB tokens. Instead of a
|
||||
bare session.get() warm-up, inject a _warm_up_session() helper
|
||||
that retries on timeout with exponential backoff.
|
||||
|
||||
Fix 2 - pdb.set_trace() crash on HTTP error
|
||||
Replace with a plain raise so errors surface cleanly.
|
||||
"""
|
||||
import re as _re
|
||||
script = SCRAPER_SCRIPT
|
||||
src = script.read_text(encoding="utf-8")
|
||||
|
||||
SENTINEL = "# patched-by-uc_salary_pipeline"
|
||||
if SENTINEL in src:
|
||||
log.info("Upstream scraper already patched.")
|
||||
return
|
||||
|
||||
log.info("Patching upstream scraper (%s) ...", script)
|
||||
|
||||
# Fix 1a: remove the stale Cookie line
|
||||
src, n1 = _re.subn(r'[ \t]*"Cookie"\s*:.*\n', "", src, count=1)
|
||||
if n1 == 0:
|
||||
log.warning("Cookie header line not found - upstream may have changed.")
|
||||
|
||||
# Fix 1b: replace bare session creation with _warm_up_session() call
|
||||
src, n2 = _re.subn(
|
||||
r'([ \t]*session = requests\.Session\(\)\n)',
|
||||
r'\1 _warm_up_session(session, base_url, request_headers)\n',
|
||||
src, count=1,
|
||||
)
|
||||
if n2 == 0:
|
||||
log.warning("session warm-up insertion point not found - upstream may have changed.")
|
||||
|
||||
# Fix 2: replace pdb.set_trace() with a plain raise
|
||||
src, n3 = _re.subn(
|
||||
r'[ \t]*import pdb\s*;\s*pdb\.set_trace\(\)\n',
|
||||
" raise\n",
|
||||
src, count=1,
|
||||
)
|
||||
if n3 == 0:
|
||||
log.warning("pdb.set_trace() not found - upstream may have changed.")
|
||||
|
||||
# Prepend sentinel + _warm_up_session helper as a plain string
|
||||
# (no triple-quote nesting issues this way)
|
||||
helper_lines = [
|
||||
'def _warm_up_session(session, base_url, headers,\n',
|
||||
' retries=5, timeout=120, backoff=15):\n',
|
||||
' # GET the UCOP wage page to obtain a fresh session cookie.\n',
|
||||
' # Retries on ReadTimeout with linear backoff.\n',
|
||||
' import time as _time\n',
|
||||
' url = base_url + "/wage/"\n',
|
||||
' for attempt in range(1, retries + 1):\n',
|
||||
' try:\n',
|
||||
' session.get(url, headers=headers, timeout=timeout)\n',
|
||||
' return\n',
|
||||
' except Exception as exc:\n',
|
||||
' if attempt == retries:\n',
|
||||
' raise RuntimeError(\n',
|
||||
' f"Could not reach {url} after {retries} attempts: {exc}"\n',
|
||||
' ) from exc\n',
|
||||
' wait = backoff * attempt\n',
|
||||
' print(f"[warm-up] attempt {attempt} failed ({exc}); "\n',
|
||||
' f"retrying in {wait}s ...")\n',
|
||||
' _time.sleep(wait)\n',
|
||||
'\n',
|
||||
]
|
||||
src = SENTINEL + "\n" + "".join(helper_lines) + src
|
||||
script.write_text(src, encoding="utf-8")
|
||||
log.info("Upstream scraper patched.")
|
||||
|
||||
|
||||
def ensure_scraper() -> None:
|
||||
"""
|
||||
Guarantee the upstream scraper repo is present and patched.
|
||||
|
||||
Steps:
|
||||
1. If UC_Employee_Salary_Data/.git is absent, git clone the repo.
|
||||
2. Apply _patch_scraper() to fix the stale cookie and pdb crash.
|
||||
3. Install requests via pip only if not already importable
|
||||
(on Arch it arrives via pacman python-requests, not pip).
|
||||
"""
|
||||
git_marker = SCRAPER_DIR / ".git"
|
||||
if not git_marker.exists():
|
||||
log.info("Scraper repo not found - cloning %s ...", SCRAPER_REPO)
|
||||
try:
|
||||
subprocess.run(
|
||||
["git", "clone", "--depth=1", SCRAPER_REPO, str(SCRAPER_DIR)],
|
||||
check=True,
|
||||
)
|
||||
except FileNotFoundError:
|
||||
log.error("git not found on PATH. Install git and retry.")
|
||||
sys.exit(1)
|
||||
except subprocess.CalledProcessError as exc:
|
||||
log.error("git clone failed: %s", exc)
|
||||
sys.exit(1)
|
||||
else:
|
||||
log.info("Scraper repo already present at %s", SCRAPER_DIR)
|
||||
|
||||
# Always call - idempotent via sentinel comment
|
||||
_patch_scraper()
|
||||
|
||||
import importlib.util
|
||||
if importlib.util.find_spec("requests") is None:
|
||||
log.info("requests not found - installing via pip ...")
|
||||
subprocess.run(
|
||||
[sys.executable, "-m", "pip", "install", "requests", "--quiet"],
|
||||
check=True,
|
||||
)
|
||||
else:
|
||||
log.info("requests already available - skipping pip install")
|
||||
|
||||
log.info("Scraper ready at %s", SCRAPER_SCRIPT)
|
||||
|
||||
|
||||
def run_scraper(year: int, output_dir: Path) -> Path:
|
||||
"""
|
||||
Run the upstream scraper for one year and return the path to the
|
||||
produced UCOP_Data_<year>.csv.
|
||||
|
||||
The upstream script writes its CSV to the process working directory,
|
||||
so we execute it with cwd=output_dir.
|
||||
|
||||
If the CSV already exists in output_dir, scraping is skipped.
|
||||
"""
|
||||
csv_path = output_dir / f"UCOP_Data_{year}.csv"
|
||||
|
||||
if csv_path.exists():
|
||||
log.info("CSV already exists for %s, skipping scrape: %s", year, csv_path)
|
||||
return csv_path
|
||||
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
log.info("Running upstream scraper for year %s ...", year)
|
||||
log.info(" Expect 20-60 minutes (220k+ rows with polite rate limiting).")
|
||||
log.info(" Upstream progress goes to ucop.log in the working directory.")
|
||||
|
||||
try:
|
||||
subprocess.run(
|
||||
[sys.executable, str(SCRAPER_SCRIPT.resolve()), "-y", str(year)],
|
||||
check=True,
|
||||
cwd=str(output_dir),
|
||||
)
|
||||
except subprocess.CalledProcessError as exc:
|
||||
log.error("Scraper failed for year %s: %s", year, exc)
|
||||
sys.exit(1)
|
||||
|
||||
if not csv_path.exists():
|
||||
log.error("Scraper finished but expected CSV not found: %s", csv_path)
|
||||
sys.exit(1)
|
||||
|
||||
log.info("Scraper produced: %s", csv_path)
|
||||
return csv_path
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CLASSIFICATION ENGINE
|
||||
# ---------------------------------------------------------------------------
|
||||
# Categories are organised around the UC Academic Senate membership rules
|
||||
# (Regents Standing Order 105.1), as listed at senate.ucla.edu/senate-membership
|
||||
#
|
||||
# SENATE_FACULTY
|
||||
# All titles that carry Academic Senate membership (excluding administrative
|
||||
# officers such as Chancellor, Dean etc. which are kept in SENIOR_MANAGEMENT):
|
||||
# - Professor series (all ranks: Full/Assoc/Asst, all pay plans: AY/FY/HCOMP/
|
||||
# B/E/E/SFT-VM/Law/1-9 etc.)
|
||||
# - Professor of Teaching / SOE / PSOE (Senate members)
|
||||
# - Professor of Clinical X (all ranks) -- Senate per SO 105.1
|
||||
# - Professor-in-Residence (all ranks) -- Senate per SO 105.1
|
||||
# - Acting Professor and Acting Professor of Teaching (all ranks) -- Senate
|
||||
# - Instructor and Instructor-in-Residence -- Senate
|
||||
# - Emeritus/Recall faculty (retired from Senate title)
|
||||
# - AES Agronomists and Astronomers (tenure series)
|
||||
#
|
||||
# NON_SENATE_ACADEMIC
|
||||
# Academic titles that do NOT carry Senate membership:
|
||||
# - Health Sciences Clinical Professor series (HS CLIN PROF) -- explicitly
|
||||
# listed as non-Senate; kept visible because their HCOMP supplements make
|
||||
# total pay structurally different from regular faculty
|
||||
# - Adjunct Professor series
|
||||
# - Visiting Professor series (including Vis Asst/Assoc Prof)
|
||||
# - Assistant Acting Professor (different from Acting Assistant Professor,
|
||||
# which IS Senate)
|
||||
# - Clinical Associate series
|
||||
# - Non-Senate Instructional / Lecturers (all variants including LECT-AY,
|
||||
# LECT-FY, LECT-AY-1/9, Summer Session, Continuing etc.)
|
||||
# - Professional Research series
|
||||
# - Project Scientist series
|
||||
# - Specialist / Cooperative Extension series
|
||||
# - Visiting Professional Research / Visiting Scholars
|
||||
# - Volunteer Clinical Professor
|
||||
# - Librarians, Academic Coordinators, Academic Administrators
|
||||
# - Postdoctoral Scholars (non-student, non-Senate)
|
||||
#
|
||||
# SENIOR_MANAGEMENT -- kept as-is (anchored patterns to avoid false positives)
|
||||
# ATHLETICS -- kept as-is (separated to avoid inflating SMG totals)
|
||||
# STUDENT_TRAINEE -- TAs, GSRs, Readers, Residents, Interns, Student workers
|
||||
# SENIOR_PROF_STAFF -- non-academic staff, gross pay >= $100k
|
||||
# PROF_STAFF -- non-academic staff, $40k <= gross pay < $100k
|
||||
# SUPPORT_STAFF -- non-academic staff, gross pay < $40k
|
||||
|
||||
CATEGORIES = [
|
||||
"SENATE_FACULTY",
|
||||
"NON_SENATE_ACADEMIC",
|
||||
"SENIOR_MANAGEMENT",
|
||||
"ATHLETICS",
|
||||
"STUDENT_TRAINEE",
|
||||
"SENIOR_PROF_STAFF",
|
||||
"PROF_STAFF",
|
||||
"SUPPORT_STAFF",
|
||||
]
|
||||
|
||||
# Pay thresholds (adjustable via CLI)
|
||||
SENIOR_STAFF_THRESHOLD = 100_000
|
||||
SUPPORT_STAFF_THRESHOLD = 40_000
|
||||
|
||||
|
||||
def _rx(*patterns):
|
||||
return [re.compile(p, re.IGNORECASE) for p in patterns]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pattern blocks — tested in the order listed in classify_title()
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# 1. STUDENT / TRAINEE
|
||||
# Tested first because some student titles contain "PROF" or "INSTR".
|
||||
# Medical residents are trainees, not staff.
|
||||
_STUDENT_RX = _rx(
|
||||
r"\bGSR\b",
|
||||
r"\bGRAD(?:UATE)? STUDENT RES",
|
||||
r"\bTEACH(?:ING)? ASST\b",
|
||||
r"[-\s]GSHIP\b",
|
||||
r"\bASSOC IN ", # Associate-in-[dept]-GSHIP student series
|
||||
r"\bREMEDIAL TUTOR\b",
|
||||
r"\bTUT\b", # Tutor (GSHIP and non-GSHIP)
|
||||
r"\bREADER\b", # Reader / Special Reader
|
||||
r"\bPOSTDOC",
|
||||
r"\bPOST[ -]?DOCTORAL",
|
||||
r"\bINTERN(?!AL)", # Intern but not Internal
|
||||
r"\bRESID(?:ENT)?\s+PHYS", # Resident Physician (medical trainees)
|
||||
r"\bPOST D\.?D\.?S\b",
|
||||
r"\bCLIN(?:ICAL)? TRAINEE\b",
|
||||
r"\bVET(?:ERINARY)?.*(?:INTERN|RESIDENT)\b",
|
||||
r"^STDT\b", # Student worker prefix
|
||||
r"^STUDENT\b", # Student 1/2/3/4, Student Researcher etc.
|
||||
r"\bSTUDENT RESEARCHER\b",
|
||||
r"\bTEACHING FELLOW\b",
|
||||
)
|
||||
|
||||
# 2. ATHLETICS
|
||||
# Tested before SMG so coaches don't match DEAN/DIR patterns.
|
||||
_ATHLETICS_RX = _rx(
|
||||
r"\bHEAD COACH\b",
|
||||
r"\bATH(?:LETIC)? COACH\b",
|
||||
r"\bINTERCOL ATH\b",
|
||||
r"\bATH DIR\b",
|
||||
r"\bATHLETICS DIR",
|
||||
r"\bATH ASSOC DIR\b",
|
||||
r"\bASC HEAD COACH\b", # Associate Head Coach (GCC abbreviation)
|
||||
r"^COACH (?:ASSISTANT|SPEC)\b", # Assistant/Specialist Coach series (2024)
|
||||
r"^RECR COACH\b", # Recreation Coach series
|
||||
r"^ATH (?:MANAGER|TRAINER|SUPERVISOR|PROFL)\b", # Athletics staff series
|
||||
r"^SPORTS MEDICAL ATH\b", # Sports Medical Athletic Trainer
|
||||
r"\bATH PROFL\b", # Ath Profl series
|
||||
)
|
||||
|
||||
|
||||
# 3. SENIOR MANAGEMENT
|
||||
# Anchored/specific patterns to avoid false positives like
|
||||
# "Assistant TO Dean" or "Director of Nursing".
|
||||
_SMG_RX = _rx(
|
||||
r"^CHANCELLOR\b",
|
||||
r"^CHANC\b",
|
||||
r"\bVICE CHANC\b",
|
||||
r"\bVICE[ -]CHANC\b",
|
||||
r"\bASSOC VICE CHANC\b",
|
||||
r"\bASST VICE CHANC\b",
|
||||
r"^PROVOST\b",
|
||||
r"\bVICE PROVOST\b",
|
||||
r"\bASSOC VICE PROVOST\b",
|
||||
r"\bASST VICE PROVOST\b",
|
||||
r"^EXEC VICE AND PROVOST\b",
|
||||
r"\bPROVOST\b", # catches Executive Vice and Provost, Senior VP variants
|
||||
r"\bVICE PRES\b",
|
||||
r"^PRESIDENT\b",
|
||||
r"^DEAN\b", # Anchored: DEAN at start, not "Assistant to Dean"
|
||||
r"^DEAN AST\b", # Assistant Dean (2015 GCC form)
|
||||
r"^DEAN ASC\b", # Associate Dean (2015 GCC form)
|
||||
r"^ACT/INTERIM DEAN\b",
|
||||
r"^DEAN UNIV EXT\b",
|
||||
r"\bDIR EXEC\b",
|
||||
r"\bEXEC DIR\b",
|
||||
r"\bEXECUTIVE DIRECTOR\b",
|
||||
r"\bLAB(?:ORATORY)? DIRECTOR\b",
|
||||
)
|
||||
|
||||
# 4. SENATE FACULTY
|
||||
# All titles conferring UC Academic Senate membership under SO 105.1,
|
||||
# excluding administrative officers (Chancellor, Dean etc.) which are in SMG.
|
||||
#
|
||||
# Key inclusions confirmed from senate.ucla.edu/senate-membership:
|
||||
# Professor (all ranks + pay plans)
|
||||
# Professor of Teaching (SOE/PSOE) -- Senate
|
||||
# Professor of Clinical X (all ranks) -- Senate (NOT the HS series)
|
||||
# Professor-in-Residence (all ranks) -- Senate
|
||||
# Acting Professor / Acting Professor of Teaching (all ranks) -- Senate
|
||||
# Instructor / Instructor-in-Residence -- Senate
|
||||
# Emeritus (retired from Senate title) / Recall Faculty
|
||||
|
||||
_SENATE_RX = _rx(
|
||||
# --- Core professorial series (all ranks, all pay plans) ---
|
||||
# Anchored at ^ to avoid matching "ADJ PROF", "VIS PROF", "HS CLIN PROF" etc.
|
||||
r"^(?:ASST\s+|ASSOC\s+)?PROF-(?:AY|FY|HCOMP|SFT-VM)\b",
|
||||
r"^(?:ASST\s+|ASSOC\s+)?PROF-AY-", # B/E/E, 1/9, 1/10, LAW variants
|
||||
r"^(?:ASST\s+|ASSOC\s+)?PROF-FY-",
|
||||
r"^(?:ASST\s+|ASSOC\s+)?PROF-SFT-VM\b",
|
||||
r"^(?:ASST\s+|ASSOC\s+)?PROF-HCOMP\b", # bare HCOMP (GCC full-prof titles)
|
||||
|
||||
# --- Professor of Teaching / SOE / PSOE (Senate) ---
|
||||
r"^(?:ASST\s+|ASSOC\s+)?PROF OF TEACH\b",
|
||||
r"^(?:ASST\s+|ASSOC\s+)?PROF OF TEACH-",
|
||||
r"PROF(?:\s+OF)?\s+TEACH.*EMERITUS",
|
||||
|
||||
# --- Professor of Clinical X (all ranks) -- Senate per SO 105.1 ---
|
||||
# NB: "HS CLIN PROF" (Health Sciences series) is NON-Senate and is
|
||||
# caught in NON_SENATE_ACADEMIC. This pattern matches the non-HS variant.
|
||||
r"^(?:ASST\s+|ASSOC\s+)?PROF OF CLIN\b",
|
||||
r"^(?:ASST\s+|ASSOC\s+)?PROF OF CLIN-",
|
||||
|
||||
# --- Professor-in-Residence (all ranks) -- Senate ---
|
||||
r"^(?:ASST\s+|ASSOC\s+)?PROF IN RES\b",
|
||||
r"^(?:ASST\s+|ASSOC\s+)?PROF IN RES-",
|
||||
|
||||
# --- Acting Professor and Acting Professor of Teaching -- Senate ---
|
||||
# "Acting Professor" IS Senate; "Assistant Acting Professor" is NOT.
|
||||
# Pattern: ACT [ASSOC|ASST] PROF (without "ASST ACT" order which = non-Senate)
|
||||
r"^ACT\s+PROF\b",
|
||||
r"^ACT\s+PROF-",
|
||||
r"^ACT\s+ASSOC\s+PROF\b",
|
||||
r"^ACT\s+ASSOC\s+PROF-",
|
||||
r"^ACT\s+ASST\s+PROF\b",
|
||||
r"^ACT\s+ASST\s+PROF-",
|
||||
r"^ACT\s+(?:ASSOC\s+|ASST\s+)?PROF OF TEACH\b",
|
||||
r"^ACT\s+(?:ASSOC\s+|ASST\s+)?PROF OF TEACH-",
|
||||
r"^ACT\s+(?:ASSOC\s+|ASST\s+)?PROF OF CLIN\b",
|
||||
r"^ACT\s+(?:ASSOC\s+|ASST\s+)?PROF OF CLIN-",
|
||||
r"^ACT\s+(?:ASSOC\s+|ASST\s+)?PROF IN RES\b",
|
||||
r"^ACT\s+PROF-AY-LAW\b", # Acting Prof Law series
|
||||
|
||||
# --- Instructor / Instructor-in-Residence -- Senate ---
|
||||
r"^INSTR\b",
|
||||
r"^INSTR-",
|
||||
r"^INSTRUCTOR\b",
|
||||
r"^(?:ASST\s+|ASSOC\s+)?INSTR IN RES\b",
|
||||
|
||||
# --- Regents' Professor (endowed Senate title) ---
|
||||
r"^REGENTS'? PROF\b",
|
||||
|
||||
# --- Emeritus / Recall (retired from Senate title) ---
|
||||
r"\bPROF(?:ESSOR)?[-\s]EMERITUS\b",
|
||||
r"\bPROF(?:ESSOR)?\s+EMERITUS\b",
|
||||
r"^RECALL FACULTY\b",
|
||||
|
||||
# --- Specialty tenure series (Senate) ---
|
||||
r"^(?:ASSOC\s+|ASST\s+)?AGRON AES(?:-AY|-FY|-B/E/E|-SFT-VM)?$",
|
||||
r"^(?:ASSOC\s+|ASST\s+)?ASTRONOMER(?:-AY|-FY)?$",
|
||||
)
|
||||
|
||||
# 5. NON-SENATE ACADEMIC
|
||||
# Academic titles WITHOUT Senate membership per SO 105.1 and senate.ucla.edu.
|
||||
_NON_SENATE_RX = _rx(
|
||||
# --- Health Sciences Clinical Professor series (explicitly non-Senate) ---
|
||||
r"^HS\s+(?:ASST\s+|ASSOC\s+)?CLIN PROF\b",
|
||||
r"^HS\s+(?:ASST\s+|ASSOC\s+)?CLIN PROF-",
|
||||
r"^HS\s+CLIN INSTR\b", # HS Clinical Instructor
|
||||
r"^HS\s+CLIN INSTR-",
|
||||
|
||||
# --- Adjunct Professor series ---
|
||||
r"\bADJ(?:UNCT)?\s+PROF\b",
|
||||
r"\bADJ\s+INSTR\b",
|
||||
|
||||
# --- Visiting Professor series (non-Senate) ---
|
||||
# "VIS PROF", "VIS ASST PROF", "VIS ASSOC PROF"
|
||||
r"\bVIS\s+(?:ASST\s+|ASSOC\s+)?PROF\b",
|
||||
r"\bVIS\s+(?:ASST\s+|ASSOC\s+)?PROF-",
|
||||
|
||||
# --- Assistant Acting Professor (non-Senate; reversed word order matters) ---
|
||||
r"^ASST\s+ACT\s+PROF\b",
|
||||
r"^ASST\s+ACT\s+PROF-",
|
||||
|
||||
# --- Clinical Associate series ---
|
||||
r"^(?:ASST\s+|ASSOC\s+)?CLIN PROF\b", # non-HS clinical (no HS prefix)
|
||||
r"^(?:ASST\s+|ASSOC\s+)?CLIN PROF-",
|
||||
r"\bCLIN PROF-VOL\b", # Volunteer Clinical Professor
|
||||
|
||||
# --- Non-Senate Instructional / Lecturers (all variants) ---
|
||||
r"\bLECTURER\b",
|
||||
r"\bLECT\b", # LECT-AY, LECT-FY, LECT-AY-1/9 etc.
|
||||
r"^LECT-",
|
||||
r"\bSR\s+LECT\b",
|
||||
r"\bSENIOR\s+LECT\b",
|
||||
r"\bLECT\s+IN\b", # Lect In Summer Session
|
||||
r"\bCONT(?:INUING)?\s+EDUCATOR\b",
|
||||
r"\bSUPV\s+OF\s+TEACH\s+ED\b",
|
||||
|
||||
# --- Professional Research series ---
|
||||
r"\bPROF(?:ESSIONAL)?\s+RES(?:EARCHER)?\b",
|
||||
r"\bSR\s+RES\b",
|
||||
r"\bASSOC\s+RES\b",
|
||||
r"^RES-(?:FY|AY)\b",
|
||||
r"^ASST\s+RES-",
|
||||
|
||||
# --- Project Scientist series ---
|
||||
r"\bPROJ(?:ECT)?\s+SCI(?:ENT)?",
|
||||
r"\bVIS\s+PROJ\s+SCI", # Visiting Project Scientist
|
||||
|
||||
# --- Specialist / Cooperative Extension series ---
|
||||
# Anchored to avoid "PHYSICIAN SPECIALIST", "IT SPECIALIST" staff titles
|
||||
r"^(?:SR\s+|ASSOC\s+|ASST\s+)?SPECIALIST(?:\s+AES|\s+COOP|\s+NEX|\s*$)",
|
||||
r"\bCOOP\s+EXT\s+(?:SPEC|ADVISOR|ADV)\b",
|
||||
|
||||
# --- Recall/Recall-type lecturer ---
|
||||
r"\bLECT-RECALL\b",
|
||||
r"\bPROF-RECALL\b", # Recalled ladder faculty (post-emeritus)
|
||||
|
||||
# --- Librarians (non-Senate academic series) ---
|
||||
r"\bLIBRARIAN\b",
|
||||
|
||||
# --- Academic Coordinators / Administrators ---
|
||||
r"\bACAD(?:EMIC)?\s+COORD\b",
|
||||
r"\bACAD(?:EMIC)?\s+ADMIN\b",
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Normalisation
|
||||
# GCC (State Controller) uses title case with full words:
|
||||
# "Assistant Prof-Ay", "Associate Prof-Ay", "Visiting Prof", "Acting Prof-Ay"
|
||||
# UCOP uses abbreviated upper case: "ASST PROF-AY", "ASSOC PROF-AY"
|
||||
# Normalise to upper case + UCOP abbreviations so all patterns work on both.
|
||||
# ---------------------------------------------------------------------------
|
||||
_GCC_EXPANSIONS = [
|
||||
(re.compile(r"\bASSISTANT\b"), "ASST"),
|
||||
(re.compile(r"\bASSOCIATE\b"), "ASSOC"),
|
||||
(re.compile(r"\bVISITING\b"), "VIS"),
|
||||
(re.compile(r"\bACTING\b"), "ACT"),
|
||||
# GCC 2024 uses "Professor-Ay" instead of "Prof-Ay".
|
||||
# Also catches the UCSB corruption "Professoressor" (doubled suffix).
|
||||
(re.compile(r"\bPROFESSOR(?:ESSOR)?\b"), "PROF"),
|
||||
]
|
||||
|
||||
|
||||
def normalise(t: str) -> str:
|
||||
"""Upper-case, collapse whitespace, expand GCC full-word rank abbreviations."""
|
||||
t = re.sub(r"\s+", " ", t.upper().strip())
|
||||
for pattern, replacement in _GCC_EXPANSIONS:
|
||||
t = pattern.sub(replacement, t)
|
||||
return t
|
||||
|
||||
|
||||
def classify_title(raw_title: str, gross_pay: float = 0.0,
|
||||
senior_threshold: float = SENIOR_STAFF_THRESHOLD,
|
||||
support_threshold: float = SUPPORT_STAFF_THRESHOLD) -> str:
|
||||
t = normalise(raw_title)
|
||||
|
||||
if any(rx.search(t) for rx in _STUDENT_RX):
|
||||
return "STUDENT_TRAINEE"
|
||||
|
||||
if any(rx.search(t) for rx in _ATHLETICS_RX):
|
||||
return "ATHLETICS"
|
||||
|
||||
if any(rx.search(t) for rx in _SMG_RX):
|
||||
return "SENIOR_MANAGEMENT"
|
||||
|
||||
# Senate faculty tested before non-Senate so that e.g. PROF OF CLIN
|
||||
# (Senate) doesn't fall through to the CLIN PROF (non-Senate) pattern.
|
||||
if any(rx.search(t) for rx in _SENATE_RX):
|
||||
return "SENATE_FACULTY"
|
||||
|
||||
if any(rx.search(t) for rx in _NON_SENATE_RX):
|
||||
return "NON_SENATE_ACADEMIC"
|
||||
|
||||
# Staff: split by pay level
|
||||
if gross_pay >= senior_threshold:
|
||||
return "SENIOR_PROF_STAFF"
|
||||
elif gross_pay >= support_threshold:
|
||||
return "PROF_STAFF"
|
||||
else:
|
||||
return "SUPPORT_STAFF"
|
||||
|
||||
|
||||
# CSV PARSING
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _dollar(s) -> float:
|
||||
try:
|
||||
return float(re.sub(r"[$,\s]", "", str(s)))
|
||||
except (ValueError, TypeError):
|
||||
return 0.0
|
||||
|
||||
|
||||
def parse_ucop_row(row: dict) -> dict | None:
|
||||
title = (row.get("title") or row.get("Title") or "").strip()
|
||||
if not title:
|
||||
return None
|
||||
gross = _dollar(row.get("gross pay") or row.get("gross_pay") or 0)
|
||||
reg = _dollar(row.get("regular pay") or row.get("regular_pay") or 0)
|
||||
over = _dollar(row.get("overtime pay") or row.get("overtime_pay") or 0)
|
||||
other = _dollar(row.get("other pay") or row.get("other_pay") or 0)
|
||||
campus = (row.get("location") or row.get("Location") or "").strip()
|
||||
gross = gross if gross > 0 else (reg + over + other)
|
||||
return {"title": title, "gross_pay": gross, "regular_pay": reg,
|
||||
"overtime_pay": over, "other_pay": other,
|
||||
"benefits": 0.0, "total_pay_benefits": gross,
|
||||
"campus": campus}
|
||||
|
||||
|
||||
def parse_tc_row(row: dict) -> dict | None:
|
||||
title = (row.get("Job Title") or row.get("Job title") or
|
||||
row.get("job_title") or "").strip()
|
||||
if not title:
|
||||
return None
|
||||
total = _dollar(row.get("Total Pay") or row.get("Total pay") or 0)
|
||||
benefits= _dollar(row.get("Benefits") or 0)
|
||||
tpb = _dollar(row.get("Total Pay & Benefits") or
|
||||
row.get("Total pay & benefits") or 0)
|
||||
if tpb == 0 and total > 0:
|
||||
tpb = total + benefits
|
||||
return {"title": title, "gross_pay": total, "regular_pay": 0.0,
|
||||
"overtime_pay": 0.0, "other_pay": 0.0,
|
||||
"benefits": benefits, "total_pay_benefits": tpb,
|
||||
"campus": ""}
|
||||
|
||||
|
||||
def parse_gcc_row(row: dict) -> dict | None:
|
||||
"""
|
||||
Parse one row from a GCC (State Controller) CSV.
|
||||
|
||||
Confirmed column names from UCB 2015:
|
||||
Position, RegularPay, OvertimePay, LumpSumPay, OtherPay, TotalWages,
|
||||
DefinedBenefitPlanContribution, HealthDentalVision,
|
||||
TotalRetirementAndHealthContribution, EmployerName
|
||||
|
||||
TotalWages = RegularPay + OvertimePay + LumpSumPay + OtherPay
|
||||
Benefits = TotalRetirementAndHealthContribution
|
||||
(= DefinedBenefitPlanContribution + HealthDentalVision)
|
||||
LumpSumPay is one-time payouts (unused vacation etc); treated as part of
|
||||
gross pay (it is included in TotalWages already).
|
||||
|
||||
Campus comes from the 'campus' column added by gcc_downloader.py when
|
||||
combining files, or from 'EmployerName' in per-campus files.
|
||||
"""
|
||||
title = (row.get("Position") or "").strip()
|
||||
if not title:
|
||||
return None
|
||||
|
||||
gross = _dollar(row.get("TotalWages") or 0)
|
||||
reg = _dollar(row.get("RegularPay") or 0)
|
||||
overtime = _dollar(row.get("OvertimePay") or 0)
|
||||
lump = _dollar(row.get("LumpSumPay") or 0)
|
||||
other = _dollar(row.get("OtherPay") or 0)
|
||||
benefits = _dollar(row.get("TotalRetirementAndHealthContribution") or 0)
|
||||
|
||||
# Fallback: recompute gross if TotalWages is blank
|
||||
if gross == 0:
|
||||
gross = reg + overtime + lump + other
|
||||
|
||||
total_pay_benefits = gross + benefits
|
||||
|
||||
# Campus: prefer the synthetic 'campus' column from the combined file,
|
||||
# fall back to EmployerName for individual per-campus files.
|
||||
campus = (row.get("campus") or row.get("EmployerName") or "").strip()
|
||||
|
||||
return {
|
||||
"title": title,
|
||||
"gross_pay": gross,
|
||||
"regular_pay": reg,
|
||||
"overtime_pay": overtime,
|
||||
"other_pay": other + lump, # lump sum folded into other
|
||||
"benefits": benefits,
|
||||
"total_pay_benefits": total_pay_benefits,
|
||||
"campus": campus,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# AGGREGATION
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def aggregate_csv(filepath: Path, source: str,
|
||||
senior_threshold: float,
|
||||
support_threshold: float) -> dict:
|
||||
totals = {
|
||||
cat: {"count": 0, "gross_pay": 0.0, "benefits": 0.0,
|
||||
"total_pay_benefits": 0.0, "other_pay": 0.0}
|
||||
for cat in CATEGORIES
|
||||
}
|
||||
unmatched: Counter = Counter()
|
||||
if source == "ucop":
|
||||
parse_fn = parse_ucop_row
|
||||
elif source == "gcc":
|
||||
parse_fn = parse_gcc_row
|
||||
else:
|
||||
parse_fn = parse_tc_row
|
||||
|
||||
with open(filepath, newline="", encoding="utf-8-sig") as fh:
|
||||
reader = csv.DictReader(fh)
|
||||
for row in reader:
|
||||
parsed = parse_fn(row)
|
||||
if parsed is None:
|
||||
continue
|
||||
cat = classify_title(
|
||||
parsed["title"],
|
||||
gross_pay=parsed["gross_pay"],
|
||||
senior_threshold=senior_threshold,
|
||||
support_threshold=support_threshold,
|
||||
)
|
||||
totals[cat]["count"] += 1
|
||||
totals[cat]["gross_pay"] += parsed["gross_pay"]
|
||||
totals[cat]["benefits"] += parsed["benefits"]
|
||||
totals[cat]["total_pay_benefits"] += parsed["total_pay_benefits"]
|
||||
totals[cat]["other_pay"] += parsed["other_pay"]
|
||||
if cat in ("SENIOR_PROF_STAFF", "PROF_STAFF", "SUPPORT_STAFF", "NON_SENATE_ACADEMIC"):
|
||||
unmatched[normalise(parsed["title"])] += 1
|
||||
|
||||
totals["_unmatched"] = unmatched
|
||||
return totals
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# OUTPUT
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
CATEGORY_LABELS = {
|
||||
"SENATE_FACULTY": "Senate Faculty",
|
||||
"NON_SENATE_ACADEMIC": "Non-Senate Academic",
|
||||
"SENIOR_MANAGEMENT": "Senior Management (SMG)",
|
||||
"ATHLETICS": "Athletics",
|
||||
"STUDENT_TRAINEE": "Student / Trainee",
|
||||
"SENIOR_PROF_STAFF": "Senior Prof/Staff (>=$100k)",
|
||||
"PROF_STAFF": "Professional Staff ($40k-$100k)",
|
||||
"SUPPORT_STAFF": "Support Staff (<$40k)",
|
||||
}
|
||||
|
||||
|
||||
def _M(v: float) -> str:
|
||||
return f"${v / 1_000_000:,.2f}M"
|
||||
|
||||
|
||||
def print_summary(year_results: dict) -> None:
|
||||
years = sorted(year_results.keys())
|
||||
cw = 24
|
||||
header = f"{'Category':<34}" + "".join(f"{y:>{cw}}" for y in years)
|
||||
sep = "=" * len(header)
|
||||
|
||||
print(f"\n{sep}")
|
||||
print("UC SALARY SUMMARY - Gross Pay (millions USD)")
|
||||
print(sep)
|
||||
print(header)
|
||||
print("-" * len(header))
|
||||
for cat in CATEGORIES:
|
||||
label = CATEGORY_LABELS[cat]
|
||||
vals = [_M(year_results[y].get(cat, {}).get("gross_pay", 0)) for y in years]
|
||||
print(f"{label:<34}" + "".join(f"{v:>{cw}}" for v in vals))
|
||||
print("-" * len(header))
|
||||
totals = [_M(sum(year_results[y].get(c, {}).get("gross_pay", 0)
|
||||
for c in CATEGORIES)) for y in years]
|
||||
print(f"{'TOTAL':<34}" + "".join(f"{v:>{cw}}" for v in totals))
|
||||
|
||||
print(f"\n{sep}")
|
||||
print("UC SALARY SUMMARY - Employee Counts")
|
||||
print(sep)
|
||||
print(header)
|
||||
print("-" * len(header))
|
||||
for cat in CATEGORIES:
|
||||
label = CATEGORY_LABELS[cat]
|
||||
vals = [f"{year_results[y].get(cat, {}).get('count', 0):,}" for y in years]
|
||||
print(f"{label:<34}" + "".join(f"{v:>{cw}}" for v in vals))
|
||||
print(sep)
|
||||
|
||||
|
||||
def write_summary_csv(year_results: dict, outpath: Path) -> None:
|
||||
with open(outpath, "w", newline="", encoding="utf-8") as fh:
|
||||
w = csv.writer(fh)
|
||||
w.writerow(["year", "category", "category_label",
|
||||
"count", "gross_pay", "benefits",
|
||||
"total_pay_benefits", "other_pay"])
|
||||
for year in sorted(year_results.keys()):
|
||||
for cat in CATEGORIES:
|
||||
d = year_results[year].get(cat, {})
|
||||
w.writerow([
|
||||
year, cat, CATEGORY_LABELS[cat],
|
||||
d.get("count", 0),
|
||||
round(d.get("gross_pay", 0), 2),
|
||||
round(d.get("benefits", 0), 2),
|
||||
round(d.get("total_pay_benefits", 0), 2),
|
||||
round(d.get("other_pay", 0), 2),
|
||||
])
|
||||
log.info("Summary CSV → %s", outpath)
|
||||
|
||||
|
||||
def write_staff_audit(year_results: dict, outpath: Path) -> None:
|
||||
"""
|
||||
Write a ranked list of every job title that landed in one of the three
|
||||
staff buckets. Useful for spotting misclassified academic titles.
|
||||
"""
|
||||
combined: Counter = Counter()
|
||||
for yr_data in year_results.values():
|
||||
combined.update(yr_data.get("_unmatched", {}))
|
||||
with open(outpath, "w", encoding="utf-8") as fh:
|
||||
fh.write("# Staff title audit - titles classified as "
|
||||
"SENIOR_PROF_STAFF / PROF_STAFF / SUPPORT_STAFF\n")
|
||||
fh.write("# count\ttitle\n\n")
|
||||
for title, cnt in combined.most_common():
|
||||
fh.write(f"{cnt}\t{title}\n")
|
||||
log.info("Staff audit → %s", outpath)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CLI
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Scrape and/or classify UC salary data into 11 categories.",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog=__doc__,
|
||||
)
|
||||
|
||||
src = parser.add_mutually_exclusive_group(required=True)
|
||||
src.add_argument("--scrape", action="store_true",
|
||||
help="Download data from UCOP for the given --years.")
|
||||
src.add_argument("--files", nargs="+", metavar="CSV",
|
||||
help="Pre-downloaded CSV files to classify.")
|
||||
|
||||
parser.add_argument("--years", nargs="+", type=int,
|
||||
help="Year(s) to scrape (required with --scrape).")
|
||||
parser.add_argument("--scrape-dir", default=".",
|
||||
help="Directory where scraped CSVs are written "
|
||||
"(default: current directory).")
|
||||
parser.add_argument("--source", choices=["ucop", "tc", "gcc"], default="ucop",
|
||||
help="CSV format: 'ucop' (default), 'tc' "
|
||||
"(Transparent California), or 'gcc' "
|
||||
"(State Controller GCC combined CSV).")
|
||||
parser.add_argument("--output", default="uc_salary_summary.csv",
|
||||
help="Output summary CSV path.")
|
||||
parser.add_argument("--staff-audit", default="staff_title_audit.txt",
|
||||
help="Output path for staff-bucket title audit.")
|
||||
parser.add_argument("--senior-threshold", type=float,
|
||||
default=SENIOR_STAFF_THRESHOLD,
|
||||
help=f"Gross pay threshold for SENIOR_PROF_STAFF "
|
||||
f"(default ${SENIOR_STAFF_THRESHOLD:,}).")
|
||||
parser.add_argument("--support-threshold", type=float,
|
||||
default=SUPPORT_STAFF_THRESHOLD,
|
||||
help=f"Gross pay below this → SUPPORT_STAFF "
|
||||
f"(default ${SUPPORT_STAFF_THRESHOLD:,}).")
|
||||
args = parser.parse_args()
|
||||
|
||||
year_results: dict = {}
|
||||
|
||||
if args.scrape:
|
||||
if not args.years:
|
||||
parser.error("--scrape requires --years")
|
||||
|
||||
# Step 1: ensure the upstream scraper repo is cloned
|
||||
ensure_scraper()
|
||||
|
||||
# Step 2: run it for each requested year; skip if CSV already exists
|
||||
scrape_dir = Path(args.scrape_dir)
|
||||
for year in args.years:
|
||||
csv_path = run_scraper(year, scrape_dir)
|
||||
log.info("Classifying %s ...", csv_path)
|
||||
year_results[str(year)] = aggregate_csv(
|
||||
csv_path, "ucop", args.senior_threshold, args.support_threshold)
|
||||
|
||||
else: # --files
|
||||
if not args.years:
|
||||
# infer from filenames
|
||||
years = []
|
||||
for f in args.files:
|
||||
m = re.search(r"\d{4}", Path(f).name)
|
||||
years.append(m.group(0) if m else Path(f).stem)
|
||||
else:
|
||||
years = [str(y) for y in args.years]
|
||||
|
||||
if len(years) != len(args.files):
|
||||
parser.error("--years count must match --files count")
|
||||
|
||||
for filepath, year in zip(args.files, years):
|
||||
p = Path(filepath)
|
||||
if not p.exists():
|
||||
log.warning("File not found, skipping: %s", p)
|
||||
continue
|
||||
log.info("Classifying %s (year=%s) …", p, year)
|
||||
result = aggregate_csv(p, args.source,
|
||||
args.senior_threshold,
|
||||
args.support_threshold)
|
||||
total_rows = sum(result[c]["count"] for c in CATEGORIES)
|
||||
log.info(" → %d rows processed", total_rows)
|
||||
year_results[year] = result
|
||||
|
||||
if not year_results:
|
||||
log.error("No data processed.")
|
||||
sys.exit(1)
|
||||
|
||||
print_summary(year_results)
|
||||
write_summary_csv(year_results, Path(args.output))
|
||||
write_staff_audit(year_results, Path(args.staff_audit))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loading…
Add table
Add a link
Reference in a new issue