mako/coldcall_lti
1
0
Fork 0
Code Activity

Phase 6: participation grading, gradebook passback, opt-out integrity

Browse Source 
Ports the timing-neutral foregone-participation grading scheme from
participation_grades.R: answer quality (per-course level points) minus
a deduction for participation foregone through unavailability,
estimated by Monte Carlo simulation of the actual weighted draw and
averaged over when absences fall, plus a small form-filing incentive
for drawn-while-absent-without-opt-out days. Reason-blind and
luck-protected; zero-answer students floor to 0. Parameters
(allowance in SD units, passing line, form penalty, simulation
size/seed) are course settings. Verified against the R engine's
rendered 2026q2 reports via the new import-legacy command: quality and
availability match exactly, finals within Monte Carlo noise; dropped
students import as inactive enrollments and are excluded identically.

Grades are computed on demand into stored GradeRun snapshots and
reviewed on a grades page with CSV export and per-student reports
(also served to students via a publish toggle). Display scales map
points to UW 4.0, a threshold table (one-click import of the Canvas
course grading scheme), or raw points. Gradebook passback via AGS
sits behind a settings toggle with a review-then-push flow.

Opt-out withdrawals are now soft-deletes with a withdrawn_at audit
trail, and close when class begins (class days gained optional start
times), so availability records cannot be rewritten after the fact.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Benjamin Mako Hill
2026-07-31 18:38:22 -07:00
parent 7ec1be5dc6
commit 46f7f55633
27 changed files with 1869 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files

123
tests/test_grades_ui.py Normal file
View File

@@ -0,0 +1,123 @@
import datetime
import re
import pytest
from conftest import outcome_actions
TODAY = datetime.date.today().isoformat()
@pytest.fixture
def instructor(dev_client):
dev_client.post("/dev/launch/dev-instructor")
# Speed: shrink the simulation for tests.
dev_client.post(
"/instructor/settings",
data={"selection_mode": "weighted", "weight_factor": "2",
"show_assessments": "on", "n_sims": "200", "sim_seed": "1"},
)
return dev_client
def record_calls(client, n=6):
for _ in range(n):
client.post("/instructor/live/next", data={"date": TODAY})
page = client.get(f"/instructor/live?date={TODAY}").get_data(as_text=True)
call_id = re.search(r"/instructor/call/(\d+)/outcome", page).group(1)
client.post(
f"/instructor/call/{call_id}/outcome",
data={"action": outcome_actions(page)["GOOD"]},
)
def test_grades_page_before_compute(instructor):
page = instructor.get("/instructor/grades/").get_data(as_text=True)
assert "No grades computed yet" in page
def test_compute_and_view_grades(instructor):
record_calls(instructor)
resp = instructor.post(
"/instructor/grades/compute", follow_redirects=True
)
page = resp.get_data(as_text=True)
assert "SD of full-attendance counts" in page
assert "4.00" in page # someone answered GOOD -> full marks
assert "never answered" in page # not everyone got called
resp = instructor.get("/instructor/grades/export.csv")
assert resp.mimetype == "text/csv"
assert "final_points" in resp.get_data(as_text=True)
def test_student_report_and_publishing(instructor):
record_calls(instructor)
instructor.post("/instructor/grades/compute")
page = instructor.get("/instructor/grades/").get_data(as_text=True)
student_id = re.search(r"/instructor/grades/student/(\d+)", page).group(1)
report = instructor.get(
f"/instructor/grades/student/{student_id}"
).get_data(as_text=True)
assert "How your grade was computed" in report
# Unpublished: students see no report link and get 404.
instructor.post("/dev/launch/dev-student-1")
assert "participation grade report" not in instructor.get("/me").get_data(as_text=True)
assert instructor.get("/me/report").status_code == 404
# Publish, and the student can read their own report.
instructor.post("/dev/launch/dev-instructor")
instructor.post(
"/instructor/settings",
data={"selection_mode": "weighted", "weight_factor": "2",
"publish_grade_reports": "on"},
)
instructor.post("/dev/launch/dev-student-1")
page = instructor.get("/me").get_data(as_text=True)
assert "participation grade report" in page
report = instructor.get("/me/report").get_data(as_text=True)
assert "How your grade was computed" in report
assert "Ada Lovelace" in report
def test_push_flow_dev_mode(instructor):
record_calls(instructor)
instructor.post("/instructor/grades/compute")
# Push pages don't exist until AGS is enabled.
assert instructor.get("/instructor/grades/push").status_code == 404
instructor.post(
"/instructor/settings",
data={"selection_mode": "weighted", "weight_factor": "2",
"ags_enabled": "on"},
)
page = instructor.get("/instructor/grades/push").get_data(as_text=True)
assert "Review before pushing" in page
assert "Push 8 grades to Canvas" in page
resp = instructor.post("/instructor/grades/push")
page = resp.get_data(as_text=True)
assert "simulated (dev mode)" in page
assert "8 of\n 8 pushed successfully" in page or "8 of" in page
def test_grades_require_instructor(dev_client):
dev_client.post("/dev/launch/dev-instructor")
dev_client.post("/dev/launch/dev-student-1")
assert dev_client.get("/instructor/grades/").status_code == 403
def test_scale_settings_and_canvas_import(instructor):
# Threshold rows via the settings form.
instructor.post(
"/instructor/settings",
data={"selection_mode": "weighted", "weight_factor": "2",
"scale_type": "table",
"scale_rows": "93 A\n90 A-\n0 F"},
)
record_calls(instructor, n=3)
instructor.post("/instructor/grades/compute")
page = instructor.get("/instructor/grades/").get_data(as_text=True)
assert "<strong>A</strong>" in page

238
tests/test_grading.py Normal file
View File

@@ -0,0 +1,238 @@
import datetime
import pytest
from coldcall_lti import grading, models, scales
D0 = datetime.date(2026, 10, 1)
def setup_course(db, n_students=6, n_days=10, calls_per_day=3,
n_sims=400, **course_kw):
course = models.Course(
lti_context_id="ctx-1", n_sims=n_sims, sim_seed=1, **course_kw
)
db.add(course)
db.flush()
models.ensure_default_levels(db, course)
levels = {
lvl.label: lvl for lvl in models.assessment_levels(db, course.id)
}
students = []
for i in range(n_students):
s = models.Student(canvas_user_id=f"u{i}", name=f"Student {i}")
db.add(s)
students.append(s)
db.flush()
for s in students:
db.add(models.Enrollment(course_id=course.id, student_id=s.id))
db.flush()
# Round-robin answered calls, all GOOD, over n_days class days.
days = [D0 + datetime.timedelta(days=i) for i in range(n_days)]
idx = 0
for day in days:
for _ in range(calls_per_day):
db.add(
models.Call(
course_id=course.id,
student_id=students[idx % n_students].id,
session_date=day,
status=models.STATUS_ANSWERED,
assessment_id=levels["GOOD"].id,
)
)
idx += 1
db.flush()
return course, students, days, levels
def test_all_present_all_good_grades_full(db_session):
course, students, days, levels = setup_course(db_session)
results = grading.compute(db_session, course)
assert results["curve"][0] == 0.0
for row in results["students"]:
assert row["q_points"] == 100.0
assert row["days_missed"] == 0
assert row["final_points"] == 100.0
def test_foregone_curve_is_nonnegative_and_grows(db_session):
course, *_ = setup_course(db_session)
results = grading.compute(db_session, course)
curve = results["curve"]
assert all(v >= 0 for v in curve)
# Missing most days must cost more than missing one.
assert curve[8] > curve[1]
def test_zero_answer_student_floors_to_zero(db_session):
course, students, days, levels = setup_course(db_session)
extra = models.Student(canvas_user_id="u-silent", name="Silent Student")
db_session.add(extra)
db_session.flush()
db_session.add(
models.Enrollment(course_id=course.id, student_id=extra.id)
)
db_session.flush()
results = grading.compute(db_session, course)
row = next(
r for r in results["students"] if r["student_id"] == extra.id
)
assert row["q_points"] is None
assert row["final_points"] == 0.0
def test_luck_protection_present_but_rarely_called(db_session):
"""A student who is always available but was called only once is
not penalized: quality is their grade."""
course, students, days, levels = setup_course(db_session)
lucky = models.Student(canvas_user_id="u-lucky", name="Lucky Student")
db_session.add(lucky)
db_session.flush()
db_session.add(
models.Enrollment(course_id=course.id, student_id=lucky.id)
)
db_session.add(
models.Call(
course_id=course.id,
student_id=lucky.id,
session_date=days[0],
status=models.STATUS_ANSWERED,
assessment_id=levels["POOR"].id,
)
)
db_session.flush()
results = grading.compute(db_session, course)
row = next(
r for r in results["students"] if r["student_id"] == lucky.id
)
assert row["days_missed"] == 0
assert row["final_points"] == pytest.approx(69.7, abs=0.01)
def test_absences_within_allowance_are_free(db_session):
course, students, days, levels = setup_course(
db_session, allowance_sd_units=5.0
)
# One student opts out of two days.
for day in days[:2]:
db_session.add(
models.OptOut(
course_id=course.id, student_id=students[0].id, date=day
)
)
db_session.flush()
results = grading.compute(db_session, course)
row = next(
r for r in results["students"]
if r["student_id"] == students[0].id
)
assert row["days_missed"] == 2
# Huge allowance: no deduction despite the absences.
assert row["final_points"] == row["q_points"]
def test_heavy_absence_is_deducted(db_session):
course, students, days, levels = setup_course(
db_session, allowance_sd_units=0.25
)
for day in days[:8]:
db_session.add(
models.OptOut(
course_id=course.id, student_id=students[0].id, date=day
)
)
db_session.flush()
results = grading.compute(db_session, course)
row = next(
r for r in results["students"]
if r["student_id"] == students[0].id
)
assert row["final_points"] < row["q_points"]
def test_form_penalty_only_for_uncovered_missing_days(db_session):
course, students, days, levels = setup_course(db_session)
caught_s, covered_s = students[0], students[1]
# Both drawn-but-missing on day 3; only covered_s filed the form.
for s in (caught_s, covered_s):
db_session.add(
models.Call(
course_id=course.id,
student_id=s.id,
session_date=days[3],
status=models.STATUS_MISSING,
)
)
db_session.add(
models.OptOut(
course_id=course.id, student_id=covered_s.id, date=days[3]
)
)
db_session.flush()
results = grading.compute(db_session, course)
rows = {r["student_id"]: r for r in results["students"]}
assert rows[caught_s.id]["caught"] == 1
assert rows[covered_s.id]["caught"] == 0
# Both have the same quality and same availability; the caught
# student ends exactly one form penalty lower.
diff = (rows[covered_s.id]["final_points"]
- rows[caught_s.id]["final_points"])
assert diff == pytest.approx(course.form_penalty_points, abs=0.01)
def test_dropped_students_are_excluded_everywhere(db_session):
course, students, days, levels = setup_course(db_session)
baseline = grading.compute(db_session, course)
dropped = models.Student(canvas_user_id="u-dropped", name="Dropped")
db_session.add(dropped)
db_session.flush()
db_session.add(
models.Enrollment(
course_id=course.id, student_id=dropped.id, active=False
)
)
for day in days[:3]:
db_session.add(
models.Call(
course_id=course.id,
student_id=dropped.id,
session_date=day,
status=models.STATUS_ANSWERED,
assessment_id=levels["GOOD"].id,
)
)
db_session.flush()
results = grading.compute(db_session, course)
assert results["excluded_students"] == 1
assert results["excluded_calls"] == 3
assert results["n_students"] == baseline["n_students"]
assert dropped.id not in {r["student_id"] for r in results["students"]}
# Same seed, same effective inputs: identical simulation results.
assert results["sd_full"] == baseline["sd_full"]
assert results["curve"] == baseline["curve"]
def test_scales():
course = models.Course(lti_context_id="x", scale_type="uw4")
assert scales.display(course, 100.0) == "4.00"
assert scales.display(course, 65.15) == "1.70"
assert scales.display(course, 84.85) == "3.00"
assert scales.display(course, 0.0) == "0.00"
course.scale_type = "table"
course.scale_config = '[[93, "A"], [90, "A-"], [0, "F"]]'
assert scales.display(course, 95) == "A"
assert scales.display(course, 91.2) == "A-"
assert scales.display(course, 10) == "F"
course.scale_type = "none"
assert scales.display(course, 87.5) == "87.5"
course.scale_type = "uw4"
assert scales.ags_score(course, 84.85) == (3.0, 4.0)
course.scale_type = "table"
assert scales.ags_score(course, 84.85) == (84.85, 100.0)

67
tests/test_student_ui.py
View File

@@ -88,6 +88,73 @@ def test_students_cannot_touch_others_optouts(student):
assert student.post(f"/me/optout/{optout_id}/delete").status_code == 404
def test_withdrawal_locks_when_class_begins(student):
# Dev class days start at 23:59, so today's class hasn't begun and
# withdrawal works (covered by test_optout_roundtrip). Move a class
# day's start time into the past and withdrawal must be refused.
from coldcall_lti.models import ClassDay
page = student.get("/me").get_data(as_text=True)
day = next_class_day_iso(page)
student.post("/me/optout", data={"date": day})
page = student.get("/me").get_data(as_text=True)
optout_id = re.search(r"/me/optout/(\d+)/delete", page).group(1)
# Reach into the DB to set that class day's start time to 00:00.
app = student.application
with app.app_context():
db = app.extensions["db_session_factory"]()
class_day = (
db.query(ClassDay)
.filter_by(date=datetime.date.fromisoformat(day))
.one()
)
began_today = class_day.date == datetime.date.today()
class_day.start_time = datetime.time(0, 0)
db.commit()
db.close()
resp = student.post(f"/me/optout/{optout_id}/delete")
if began_today:
assert resp.status_code == 400
page = student.get("/me").get_data(as_text=True)
assert "locked" in page
else:
# The day is still in the future; withdrawal stays open.
assert resp.status_code == 302
def test_withdrawal_is_recorded_not_deleted(student):
page = student.get("/me").get_data(as_text=True)
day = next_class_day_iso(page)
student.post("/me/optout", data={"date": day})
page = student.get("/me").get_data(as_text=True)
optout_id = re.search(r"/me/optout/(\d+)/delete", page).group(1)
student.post(f"/me/optout/{optout_id}/delete")
from coldcall_lti.models import OptOut
app = student.application
with app.app_context():
db = app.extensions["db_session_factory"]()
optout = db.query(OptOut).one()
assert optout.withdrawn_at is not None
db.close()
# Re-opting out reinstates the same row.
student.post("/me/optout", data={"date": day})
with app.app_context():
db = app.extensions["db_session_factory"]()
optout = db.query(OptOut).one()
assert optout.withdrawn_at is None
db.close()
# The instructor's opt-out export keeps the audit column.
student.post("/dev/launch/dev-instructor")
csv_text = student.get("/instructor/export/optouts.csv").get_data(as_text=True)
assert "withdrawn_at" in csv_text.splitlines()[0]
def test_assessment_visibility_setting(student):
# Record an answered call for dev-student-1 as the instructor.
student.post("/dev/launch/dev-instructor")