basic refactor, UCOP-wage data, classifier, csv output
This commit is contained in:
Miloš Jovanović
parent
9fafcb649f
commit
b72a2fd707
101 changed files with 693 additions and 3386246 deletions
360
classifier.py
Normal file
360
classifier.py
Normal file
|
|
@ -0,0 +1,360 @@
|
|||
"""
|
||||
classifier.py
|
||||
==============
|
||||
UC employee title classification engine: sorts every employee into one of
|
||||
8 analytical categories based on job title (and, for non-academic staff,
|
||||
gross pay). Independent of any particular data source — feed it a raw
|
||||
title string and (optionally) a gross pay figure.
|
||||
|
||||
CATEGORIES (8 total)
|
||||
---------------------
|
||||
Categories are organised around the UC Academic Senate membership rules
|
||||
(Regents Standing Order 105.1), as listed at senate.ucla.edu/senate-membership
|
||||
|
||||
SENATE_FACULTY
|
||||
All titles that carry Academic Senate membership (excluding administrative
|
||||
officers such as Chancellor, Dean etc. which are kept in SENIOR_MANAGEMENT):
|
||||
- Professor series (all ranks: Full/Assoc/Asst, all pay plans: AY/FY/HCOMP/
|
||||
B/E/E/SFT-VM/Law/1-9 etc.)
|
||||
- Professor of Teaching / SOE / PSOE (Senate members)
|
||||
- Professor of Clinical X (all ranks) -- Senate per SO 105.1
|
||||
- Professor-in-Residence (all ranks) -- Senate per SO 105.1
|
||||
- Acting Professor and Acting Professor of Teaching (all ranks) -- Senate
|
||||
- Instructor and Instructor-in-Residence -- Senate
|
||||
- Emeritus/Recall faculty (retired from Senate title)
|
||||
- AES Agronomists and Astronomers (tenure series)
|
||||
|
||||
NON_SENATE_ACADEMIC
|
||||
Academic titles that do NOT carry Senate membership:
|
||||
- Health Sciences Clinical Professor series (HS CLIN PROF) -- explicitly
|
||||
listed as non-Senate; kept visible because their HCOMP supplements make
|
||||
total pay structurally different from regular faculty
|
||||
- Adjunct Professor series
|
||||
- Visiting Professor series (including Vis Asst/Assoc Prof)
|
||||
- Assistant Acting Professor (different from Acting Assistant Professor,
|
||||
which IS Senate)
|
||||
- Clinical Associate series
|
||||
- Non-Senate Instructional / Lecturers (all variants including LECT-AY,
|
||||
LECT-FY, LECT-AY-1/9, Summer Session, Continuing etc.)
|
||||
- Professional Research series
|
||||
- Project Scientist series
|
||||
- Specialist / Cooperative Extension series
|
||||
- Visiting Professional Research / Visiting Scholars
|
||||
- Volunteer Clinical Professor
|
||||
- Librarians, Academic Coordinators, Academic Administrators
|
||||
- Postdoctoral Scholars (non-student, non-Senate)
|
||||
|
||||
SENIOR_MANAGEMENT -- kept as-is (anchored patterns to avoid false positives)
|
||||
ATHLETICS -- kept as-is (separated to avoid inflating SMG totals)
|
||||
STUDENT_TRAINEE -- TAs, GSRs, Readers, Residents, Interns, Student workers
|
||||
SENIOR_PROF_STAFF -- non-academic staff, gross pay >= $100k
|
||||
PROF_STAFF -- non-academic staff, $40k <= gross pay < $100k
|
||||
SUPPORT_STAFF -- non-academic staff, gross pay < $40k
|
||||
|
||||
Pay thresholds for SENIOR_PROF_STAFF vs PROF_STAFF vs SUPPORT_STAFF are
|
||||
applied to gross_pay and can be overridden per-call via classify_title()'s
|
||||
senior_threshold / support_threshold arguments.
|
||||
"""
|
||||
|
||||
import re
|
||||
|
||||
CATEGORIES = [
|
||||
"SENATE_FACULTY",
|
||||
"NON_SENATE_ACADEMIC",
|
||||
"SENIOR_MANAGEMENT",
|
||||
"ATHLETICS",
|
||||
"STUDENT_TRAINEE",
|
||||
"SENIOR_PROF_STAFF",
|
||||
"PROF_STAFF",
|
||||
"SUPPORT_STAFF",
|
||||
]
|
||||
|
||||
CATEGORY_LABELS = {
|
||||
"SENATE_FACULTY": "Senate Faculty",
|
||||
"NON_SENATE_ACADEMIC": "Non-Senate Academic",
|
||||
"SENIOR_MANAGEMENT": "Senior Management (SMG)",
|
||||
"ATHLETICS": "Athletics",
|
||||
"STUDENT_TRAINEE": "Student / Trainee",
|
||||
"SENIOR_PROF_STAFF": "Senior Prof/Staff (>=$100k)",
|
||||
"PROF_STAFF": "Professional Staff ($40k-$100k)",
|
||||
"SUPPORT_STAFF": "Support Staff (<$40k)",
|
||||
}
|
||||
|
||||
# Pay thresholds (adjustable per-call)
|
||||
SENIOR_STAFF_THRESHOLD = 100_000
|
||||
SUPPORT_STAFF_THRESHOLD = 40_000
|
||||
|
||||
|
||||
def _rx(*patterns):
|
||||
return [re.compile(p, re.IGNORECASE) for p in patterns]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pattern blocks — tested in the order listed in classify_title()
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# 1. STUDENT / TRAINEE
|
||||
# Tested first because some student titles contain "PROF" or "INSTR".
|
||||
# Medical residents are trainees, not staff.
|
||||
_STUDENT_RX = _rx(
|
||||
r"\bGSR\b",
|
||||
r"\bGRAD(?:UATE)? STUDENT RES",
|
||||
r"\bTEACH(?:ING)? ASST\b",
|
||||
r"[-\s]GSHIP\b",
|
||||
r"\bASSOC IN ", # Associate-in-[dept]-GSHIP student series
|
||||
r"\bREMEDIAL TUTOR\b",
|
||||
r"\bTUT\b", # Tutor (GSHIP and non-GSHIP)
|
||||
r"\bREADER\b", # Reader / Special Reader
|
||||
r"\bPOSTDOC",
|
||||
r"\bPOST[ -]?DOCTORAL",
|
||||
r"\bINTERN(?!AL)", # Intern but not Internal
|
||||
r"\bRESID(?:ENT)?\s+PHYS", # Resident Physician (medical trainees)
|
||||
r"\bPOST D\.?D\.?S\b",
|
||||
r"\bCLIN(?:ICAL)? TRAINEE\b",
|
||||
r"\bVET(?:ERINARY)?.*(?:INTERN|RESIDENT)\b",
|
||||
r"^STDT\b", # Student worker prefix
|
||||
r"^STUDENT\b", # Student 1/2/3/4, Student Researcher etc.
|
||||
r"\bSTUDENT RESEARCHER\b",
|
||||
r"\bTEACHING FELLOW\b",
|
||||
)
|
||||
|
||||
# 2. ATHLETICS
|
||||
# Tested before SMG so coaches don't match DEAN/DIR patterns.
|
||||
_ATHLETICS_RX = _rx(
|
||||
r"\bHEAD COACH\b",
|
||||
r"\bATH(?:LETIC)? COACH\b",
|
||||
r"\bINTERCOL ATH\b",
|
||||
r"\bATH DIR\b",
|
||||
r"\bATHLETICS DIR",
|
||||
r"\bATH ASSOC DIR\b",
|
||||
r"\bASC HEAD COACH\b", # Associate Head Coach (GCC abbreviation)
|
||||
r"^COACH (?:ASSISTANT|SPEC)\b", # Assistant/Specialist Coach series (2024)
|
||||
r"^RECR COACH\b", # Recreation Coach series
|
||||
r"^ATH (?:MANAGER|TRAINER|SUPERVISOR|PROFL)\b", # Athletics staff series
|
||||
r"^SPORTS MEDICAL ATH\b", # Sports Medical Athletic Trainer
|
||||
r"\bATH PROFL\b", # Ath Profl series
|
||||
)
|
||||
|
||||
|
||||
# 3. SENIOR MANAGEMENT
|
||||
# Anchored/specific patterns to avoid false positives like
|
||||
# "Assistant TO Dean" or "Director of Nursing".
|
||||
_SMG_RX = _rx(
|
||||
r"^CHANCELLOR\b",
|
||||
r"^CHANC\b",
|
||||
r"\bVICE CHANC\b",
|
||||
r"\bVICE[ -]CHANC\b",
|
||||
r"\bASSOC VICE CHANC\b",
|
||||
r"\bASST VICE CHANC\b",
|
||||
r"^PROVOST\b",
|
||||
r"\bVICE PROVOST\b",
|
||||
r"\bASSOC VICE PROVOST\b",
|
||||
r"\bASST VICE PROVOST\b",
|
||||
r"^EXEC VICE AND PROVOST\b",
|
||||
r"\bPROVOST\b", # catches Executive Vice and Provost, Senior VP variants
|
||||
r"\bVICE PRES\b",
|
||||
r"^PRESIDENT\b",
|
||||
r"^DEAN\b", # Anchored: DEAN at start, not "Assistant to Dean"
|
||||
r"^DEAN AST\b", # Assistant Dean (2015 GCC form)
|
||||
r"^DEAN ASC\b", # Associate Dean (2015 GCC form)
|
||||
r"^ACT/INTERIM DEAN\b",
|
||||
r"^DEAN UNIV EXT\b",
|
||||
r"\bDIR EXEC\b",
|
||||
r"\bEXEC DIR\b",
|
||||
r"\bEXECUTIVE DIRECTOR\b",
|
||||
r"\bLAB(?:ORATORY)? DIRECTOR\b",
|
||||
)
|
||||
|
||||
# 4. SENATE FACULTY
|
||||
# All titles conferring UC Academic Senate membership under SO 105.1,
|
||||
# excluding administrative officers (Chancellor, Dean etc.) which are in SMG.
|
||||
#
|
||||
# Key inclusions confirmed from senate.ucla.edu/senate-membership:
|
||||
# Professor (all ranks + pay plans)
|
||||
# Professor of Teaching (SOE/PSOE) -- Senate
|
||||
# Professor of Clinical X (all ranks) -- Senate (NOT the HS series)
|
||||
# Professor-in-Residence (all ranks) -- Senate
|
||||
# Acting Professor / Acting Professor of Teaching (all ranks) -- Senate
|
||||
# Instructor / Instructor-in-Residence -- Senate
|
||||
# Emeritus (retired from Senate title) / Recall Faculty
|
||||
|
||||
_SENATE_RX = _rx(
|
||||
# --- Core professorial series (all ranks, all pay plans) ---
|
||||
# Anchored at ^ to avoid matching "ADJ PROF", "VIS PROF", "HS CLIN PROF" etc.
|
||||
r"^(?:ASST\s+|ASSOC\s+)?PROF-(?:AY|FY|HCOMP|SFT-VM)\b",
|
||||
r"^(?:ASST\s+|ASSOC\s+)?PROF-AY-", # B/E/E, 1/9, 1/10, LAW variants
|
||||
r"^(?:ASST\s+|ASSOC\s+)?PROF-FY-",
|
||||
r"^(?:ASST\s+|ASSOC\s+)?PROF-SFT-VM\b",
|
||||
r"^(?:ASST\s+|ASSOC\s+)?PROF-HCOMP\b", # bare HCOMP (GCC full-prof titles)
|
||||
|
||||
# --- Professor of Teaching / SOE / PSOE (Senate) ---
|
||||
r"^(?:ASST\s+|ASSOC\s+)?PROF OF TEACH\b",
|
||||
r"^(?:ASST\s+|ASSOC\s+)?PROF OF TEACH-",
|
||||
r"PROF(?:\s+OF)?\s+TEACH.*EMERITUS",
|
||||
|
||||
# --- Professor of Clinical X (all ranks) -- Senate per SO 105.1 ---
|
||||
# NB: "HS CLIN PROF" (Health Sciences series) is NON-Senate and is
|
||||
# caught in NON_SENATE_ACADEMIC. This pattern matches the non-HS variant.
|
||||
r"^(?:ASST\s+|ASSOC\s+)?PROF OF CLIN\b",
|
||||
r"^(?:ASST\s+|ASSOC\s+)?PROF OF CLIN-",
|
||||
|
||||
# --- Professor-in-Residence (all ranks) -- Senate ---
|
||||
r"^(?:ASST\s+|ASSOC\s+)?PROF IN RES\b",
|
||||
r"^(?:ASST\s+|ASSOC\s+)?PROF IN RES-",
|
||||
|
||||
# --- Acting Professor and Acting Professor of Teaching -- Senate ---
|
||||
# "Acting Professor" IS Senate; "Assistant Acting Professor" is NOT.
|
||||
# Pattern: ACT [ASSOC|ASST] PROF (without "ASST ACT" order which = non-Senate)
|
||||
r"^ACT\s+PROF\b",
|
||||
r"^ACT\s+PROF-",
|
||||
r"^ACT\s+ASSOC\s+PROF\b",
|
||||
r"^ACT\s+ASSOC\s+PROF-",
|
||||
r"^ACT\s+ASST\s+PROF\b",
|
||||
r"^ACT\s+ASST\s+PROF-",
|
||||
r"^ACT\s+(?:ASSOC\s+|ASST\s+)?PROF OF TEACH\b",
|
||||
r"^ACT\s+(?:ASSOC\s+|ASST\s+)?PROF OF TEACH-",
|
||||
r"^ACT\s+(?:ASSOC\s+|ASST\s+)?PROF OF CLIN\b",
|
||||
r"^ACT\s+(?:ASSOC\s+|ASST\s+)?PROF OF CLIN-",
|
||||
r"^ACT\s+(?:ASSOC\s+|ASST\s+)?PROF IN RES\b",
|
||||
r"^ACT\s+PROF-AY-LAW\b", # Acting Prof Law series
|
||||
|
||||
# --- Instructor / Instructor-in-Residence -- Senate ---
|
||||
r"^INSTR\b",
|
||||
r"^INSTR-",
|
||||
r"^INSTRUCTOR\b",
|
||||
r"^(?:ASST\s+|ASSOC\s+)?INSTR IN RES\b",
|
||||
|
||||
# --- Regents' Professor (endowed Senate title) ---
|
||||
r"^REGENTS'? PROF\b",
|
||||
|
||||
# --- Emeritus / Recall (retired from Senate title) ---
|
||||
r"\bPROF(?:ESSOR)?[-\s]EMERITUS\b",
|
||||
r"\bPROF(?:ESSOR)?\s+EMERITUS\b",
|
||||
r"^RECALL FACULTY\b",
|
||||
|
||||
# --- Specialty tenure series (Senate) ---
|
||||
r"^(?:ASSOC\s+|ASST\s+)?AGRON AES(?:-AY|-FY|-B/E/E|-SFT-VM)?$",
|
||||
r"^(?:ASSOC\s+|ASST\s+)?ASTRONOMER(?:-AY|-FY)?$",
|
||||
)
|
||||
|
||||
# 5. NON-SENATE ACADEMIC
|
||||
# Academic titles WITHOUT Senate membership per SO 105.1 and senate.ucla.edu.
|
||||
_NON_SENATE_RX = _rx(
|
||||
# --- Health Sciences Clinical Professor series (explicitly non-Senate) ---
|
||||
r"^HS\s+(?:ASST\s+|ASSOC\s+)?CLIN PROF\b",
|
||||
r"^HS\s+(?:ASST\s+|ASSOC\s+)?CLIN PROF-",
|
||||
r"^HS\s+CLIN INSTR\b", # HS Clinical Instructor
|
||||
r"^HS\s+CLIN INSTR-",
|
||||
|
||||
# --- Adjunct Professor series ---
|
||||
r"\bADJ(?:UNCT)?\s+PROF\b",
|
||||
r"\bADJ\s+INSTR\b",
|
||||
|
||||
# --- Visiting Professor series (non-Senate) ---
|
||||
# "VIS PROF", "VIS ASST PROF", "VIS ASSOC PROF"
|
||||
r"\bVIS\s+(?:ASST\s+|ASSOC\s+)?PROF\b",
|
||||
r"\bVIS\s+(?:ASST\s+|ASSOC\s+)?PROF-",
|
||||
|
||||
# --- Assistant Acting Professor (non-Senate; reversed word order matters) ---
|
||||
r"^ASST\s+ACT\s+PROF\b",
|
||||
r"^ASST\s+ACT\s+PROF-",
|
||||
|
||||
# --- Clinical Associate series ---
|
||||
r"^(?:ASST\s+|ASSOC\s+)?CLIN PROF\b", # non-HS clinical (no HS prefix)
|
||||
r"^(?:ASST\s+|ASSOC\s+)?CLIN PROF-",
|
||||
r"\bCLIN PROF-VOL\b", # Volunteer Clinical Professor
|
||||
|
||||
# --- Non-Senate Instructional / Lecturers (all variants) ---
|
||||
r"\bLECTURER\b",
|
||||
r"\bLECT\b", # LECT-AY, LECT-FY, LECT-AY-1/9 etc.
|
||||
r"^LECT-",
|
||||
r"\bSR\s+LECT\b",
|
||||
r"\bSENIOR\s+LECT\b",
|
||||
r"\bLECT\s+IN\b", # Lect In Summer Session
|
||||
r"\bCONT(?:INUING)?\s+EDUCATOR\b",
|
||||
r"\bSUPV\s+OF\s+TEACH\s+ED\b",
|
||||
|
||||
# --- Professional Research series ---
|
||||
r"\bPROF(?:ESSIONAL)?\s+RES(?:EARCHER)?\b",
|
||||
r"\bSR\s+RES\b",
|
||||
r"\bASSOC\s+RES\b",
|
||||
r"^RES-(?:FY|AY)\b",
|
||||
r"^ASST\s+RES-",
|
||||
|
||||
# --- Project Scientist series ---
|
||||
r"\bPROJ(?:ECT)?\s+SCI(?:ENT)?",
|
||||
r"\bVIS\s+PROJ\s+SCI", # Visiting Project Scientist
|
||||
|
||||
# --- Specialist / Cooperative Extension series ---
|
||||
# Anchored to avoid "PHYSICIAN SPECIALIST", "IT SPECIALIST" staff titles
|
||||
r"^(?:SR\s+|ASSOC\s+|ASST\s+)?SPECIALIST(?:\s+AES|\s+COOP|\s+NEX|\s*$)",
|
||||
r"\bCOOP\s+EXT\s+(?:SPEC|ADVISOR|ADV)\b",
|
||||
|
||||
# --- Recall/Recall-type lecturer ---
|
||||
r"\bLECT-RECALL\b",
|
||||
r"\bPROF-RECALL\b", # Recalled ladder faculty (post-emeritus)
|
||||
|
||||
# --- Librarians (non-Senate academic series) ---
|
||||
r"\bLIBRARIAN\b",
|
||||
|
||||
# --- Academic Coordinators / Administrators ---
|
||||
r"\bACAD(?:EMIC)?\s+COORD\b",
|
||||
r"\bACAD(?:EMIC)?\s+ADMIN\b",
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Normalisation
|
||||
# GCC (State Controller) uses title case with full words:
|
||||
# "Assistant Prof-Ay", "Associate Prof-Ay", "Visiting Prof", "Acting Prof-Ay"
|
||||
# UCOP uses abbreviated upper case: "ASST PROF-AY", "ASSOC PROF-AY"
|
||||
# Normalise to upper case + UCOP abbreviations so all patterns work on both.
|
||||
# ---------------------------------------------------------------------------
|
||||
_GCC_EXPANSIONS = [
|
||||
(re.compile(r"\bASSISTANT\b"), "ASST"),
|
||||
(re.compile(r"\bASSOCIATE\b"), "ASSOC"),
|
||||
(re.compile(r"\bVISITING\b"), "VIS"),
|
||||
(re.compile(r"\bACTING\b"), "ACT"),
|
||||
# GCC 2024 uses "Professor-Ay" instead of "Prof-Ay".
|
||||
# Also catches the UCSB corruption "Professoressor" (doubled suffix).
|
||||
(re.compile(r"\bPROFESSOR(?:ESSOR)?\b"), "PROF"),
|
||||
]
|
||||
|
||||
|
||||
def normalise(t: str) -> str:
|
||||
"""Upper-case, collapse whitespace, expand GCC full-word rank abbreviations."""
|
||||
t = re.sub(r"\s+", " ", t.upper().strip())
|
||||
for pattern, replacement in _GCC_EXPANSIONS:
|
||||
t = pattern.sub(replacement, t)
|
||||
return t
|
||||
|
||||
|
||||
def classify_title(raw_title: str, gross_pay: float = 0.0,
|
||||
senior_threshold: float = SENIOR_STAFF_THRESHOLD,
|
||||
support_threshold: float = SUPPORT_STAFF_THRESHOLD) -> str:
|
||||
t = normalise(raw_title)
|
||||
|
||||
if any(rx.search(t) for rx in _STUDENT_RX):
|
||||
return "STUDENT_TRAINEE"
|
||||
|
||||
if any(rx.search(t) for rx in _ATHLETICS_RX):
|
||||
return "ATHLETICS"
|
||||
|
||||
if any(rx.search(t) for rx in _SMG_RX):
|
||||
return "SENIOR_MANAGEMENT"
|
||||
|
||||
# Senate faculty tested before non-Senate so that e.g. PROF OF CLIN
|
||||
# (Senate) doesn't fall through to the CLIN PROF (non-Senate) pattern.
|
||||
if any(rx.search(t) for rx in _SENATE_RX):
|
||||
return "SENATE_FACULTY"
|
||||
|
||||
if any(rx.search(t) for rx in _NON_SENATE_RX):
|
||||
return "NON_SENATE_ACADEMIC"
|
||||
|
||||
# Staff: split by pay level
|
||||
if gross_pay >= senior_threshold:
|
||||
return "SENIOR_PROF_STAFF"
|
||||
elif gross_pay >= support_threshold:
|
||||
return "PROF_STAFF"
|
||||
else:
|
||||
return "SUPPORT_STAFF"
|
||||
Loading…
Add table
Add a link
Reference in a new issue