#!/usr/bin/env python3
"""Extract course (CLO) and module (MLO) learning objectives from Canvas.

CLOs live in the syllabus (syllabus endpoint, or an attached docx/pdf -- this
module reads the endpoint; attachment parsing is a later path). MLOs live in
module overview pages. Both are Bloom-classified so the assurance inventory can
ground each assessment's cognitive level in its module's MLOs instead of relying
on the assessment title alone.

Design contracts
----------------
- Pure read. No Canvas writes.
- Graceful degrade: any fetch/parse failure returns an empty result; callers
  simply get no objective grounding (same as the prior title-only behavior).
- No em or en dashes in string literals (house rule).
- Bloom classification reuses scripts/blooms.classify_blooms -- one source of truth.

Public API
----------
extract_course_clos(config) -> dict
    {"clos": [str], "bloom_index": int, "bloom_level": str, "source": str}

extract_module_mlos(config, *, tree=None) -> dict
    {module_name: {"mlos": [str], "bloom_index": int, "bloom_level": str}}
"""

from __future__ import annotations

import logging
import os
import re
import sys
from html import unescape

_SCRIPTS_DIR = os.path.dirname(os.path.abspath(__file__))
if _SCRIPTS_DIR not in sys.path:
    sys.path.insert(0, _SCRIPTS_DIR)

import canvas_api
from blooms import classify_blooms, BLOOMS_LEVELS

logger = logging.getLogger(__name__)

# Headings that introduce a course-outcome list in a syllabus.
_CLO_MARKERS = (
    "course learning outcomes",
    "learning outcomes",
    "course outcomes",
    "student learning outcomes",
)

# Phrase that typically precedes the outcome list ("...will be able to:").
_ABLE_TO = "able to"

# MLO line pattern: "MLO 1.2 Create and run R code ..." (tolerant of spacing/punct).
# Anchored to a line start (MULTILINE ^) so an inline prose reference such as
# "...aligns with MLO 1.1..." is NOT captured as a fake MLO -- only a token that
# BEGINS a line/list item counts. Run against _block_text(), which preserves
# block boundaries as newlines (plain _strip_html flattens them away).
_MLO_RE = re.compile(
    r"^[ \t]*MLO\s*\d+\.\d+[:.\)\s]+([A-Za-z].*?)(?=^[ \t]*MLO\s*\d+\.\d+|\Z)",
    re.IGNORECASE | re.DOTALL | re.MULTILINE,
)

# Fallback (no "MLO N.N" tokens): a heading that introduces a plain-<li> objective
# list on a module overview page. Many ASU courses (e.g. COM 100) list module
# objectives as bullets under a "Learning Objectives" heading, sub-grouped by
# textbook chapter, with no "MLO N.N" prefix. Markers are matched against short
# heading-block text only (not raw HTML, not <li> text), so a marker word inside a
# normal prose sentence does not false-anchor the harvest. A real <h> anchor is
# preferred over a <p> one; a very short <p> that is essentially just the marker
# can still anchor (the same heuristic that lets <p> pseudo-headings work).
_MLO_HEADING_MARKERS = (
    "learning objectives",
    "learning objective",
    "module objectives",
    "objectives",
    "outcomes",
)

# Lists under a "Self-check" heading are logistics checklists (read chapters,
# take the quiz), not objectives. Matched against HEADING TEXT only -- a bullet
# that merely contains "self-check" must NOT truncate the harvest.
_SELF_CHECK_MARKERS = ("self-check", "self check")

# Heading-like blocks: real <h1-6> tags AND short <p> "pseudo-headings" (some
# overview pages, e.g. COM 100 Module 7, style "Learning Objectives" as a large
# <p><span> instead of a real heading). A heading label is short; the length cap
# keeps a marker word buried in a prose sentence from anchoring the harvest.
_BLOCK_RE = re.compile(r"<(h[1-6]|p)\b[^>]*>(.*?)</\1>", re.IGNORECASE | re.DOTALL)
_MAX_HEADING_LEN = 40


def _strip_html(html: str) -> str:
    """Collapse HTML to plain text (tags removed, entities unescaped, ws normalized)."""
    text = re.sub(r"<[^>]+>", " ", html or "")
    return re.sub(r"\s+", " ", unescape(text)).strip()


def _block_text(html: str) -> str:
    """Plain text with block boundaries preserved as newlines.

    Like _strip_html but each <li>/<p>/<div>/<br>/<tr>/<h*> boundary becomes a
    newline, so a line that BEGINS with an MLO token can be told apart from an
    inline prose mention of one. Only spaces/tabs are collapsed; newlines (the
    original source breaks too) survive.
    """
    if not html:
        return ""
    html = re.sub(r"(?i)<br\s*/?>|</(?:li|p|div|h[1-6]|tr|td|th)\s*>|<li\b[^>]*>", "\n", html)
    text = unescape(re.sub(r"<[^>]+>", " ", html))
    text = re.sub(r"[ \t]+", " ", text)
    lines = (ln.strip() for ln in text.splitlines())
    return "\n".join(ln for ln in lines if ln)


def _list_items(html: str) -> list[str]:
    """Return the text of every <li> in the HTML, in order."""
    return [_strip_html(m) for m in re.findall(r"<li\b[^>]*>(.*?)</li>", html or "", re.IGNORECASE | re.DOTALL)]


def _classify(statements: list[str]) -> tuple[int, str]:
    """Return (max_bloom_index, level_name) across the objective statements.

    Each statement is classified by its leading Bloom verb (classify_blooms reads
    the first recognized verb). The set's level is the MAX (a module that asks
    students to Create is a Create module even if some MLOs are Understand).
    Empty / all-unknown -> (0, "Unknown").
    """
    best = 0
    for s in statements:
        _, level = classify_blooms(s)
        idx = BLOOMS_LEVELS.get(level, 0)
        if idx > best:
            best = idx
    if best == 0:
        return 0, "Unknown"
    name = next(k for k, v in BLOOMS_LEVELS.items() if v == best)
    return best, name


def _outcomes_from_syllabus_html(html: str) -> list[str]:
    """Extract the course-outcome statements from syllabus HTML.

    Strategy: locate the outcomes heading, then take the <li> items that follow
    it (within a window). If no list is present, fall back to splitting the
    "...will be able to:" clause on sentence boundaries.
    """
    if not html:
        return []
    lower = html.lower()

    # Find the earliest outcomes marker.
    pos = -1
    for marker in _CLO_MARKERS:
        i = lower.find(marker)
        if i >= 0 and (pos < 0 or i < pos):
            pos = i
    if pos < 0:
        return []

    window = html[pos:pos + 6000]
    # The CLO list is the FIRST <ul>/<ol> after the heading. Taking only that one
    # block avoids sweeping in later sections (Technology Requirements, Texts, etc.)
    # whose own <li> items would otherwise pollute the CLO set.
    block = re.search(r"<(ul|ol)\b[^>]*>(.*?)</\1>", window, re.IGNORECASE | re.DOTALL)
    if block:
        items = [it for it in _list_items(block.group(0)) if 0 < len(it) <= 400]
        if items:
            return items

    # No <li>: fall back to the "able to:" clause split on sentence enders.
    text = _strip_html(window)
    low = text.lower()
    a = low.find(_ABLE_TO)
    if a < 0:
        return []
    tail = text[a + len(_ABLE_TO):].lstrip(": ").strip()
    # Split on sentence boundaries; keep non-trivial fragments.
    parts = re.split(r"(?<=[.;])\s+", tail)
    return [p.strip(" .;") for p in parts[:12] if len(p.strip()) > 12]


def _mlos_from_heading_list(html: str) -> list[str]:
    """Fallback MLO harvest for overview pages with no "MLO N.N" tokens.

    Locate the first heading-like block whose text names an objectives list
    ("Learning Objectives" / "module objectives" / "objectives" / "outcomes") and
    return the <li> items inside that section -- across any chapter sub-headings.
    The section ends at the next "Self-check" heading, or (for a real <h>) the next
    same-or-higher-level heading, so logistics checklists never bleed in. Anchoring
    on short heading-like blocks (not raw HTML) prevents a marker word buried in a
    prose sentence or a <li> from false-anchoring. Empty if no such heading.
    """
    if not html:
        return []

    # Short heading-like blocks: (start, end, level_or_None, text_lower).
    blocks = []
    for m in _BLOCK_RE.finditer(html):
        text = _strip_html(m.group(2))
        if not text or len(text) > _MAX_HEADING_LEN:
            continue
        tag = m.group(1).lower()
        level = int(tag[1]) if tag.startswith("h") else None
        blocks.append((m.start(), m.end(), level, text.lower()))

    # Prefer a real <h> heading anchor; fall back to a <p> pseudo-heading only if
    # no real heading names the list. Keeps a short stray <p> ("Our objectives:")
    # from out-anchoring a genuine Learning Objectives <h2>.
    def _names_objectives(b):
        return any(mk in b[3] for mk in _MLO_HEADING_MARKERS)

    obj = next((b for b in blocks if b[2] is not None and _names_objectives(b)), None)
    if obj is None:
        obj = next((b for b in blocks if _names_objectives(b)), None)
    if obj is None:
        return []

    _, obj_end, obj_level, _ = obj
    # Section runs until the next Self-check heading, or (real <h> only) the next
    # same-or-higher-level heading. Cap as a runaway backstop.
    # ponytail: pseudo-heading (<p>) sections rely on the Self-check cut alone; a
    # trailing non-self-check section could over-harvest. Not seen in the data.
    sec_end = min(len(html), obj_end + 8000)
    for b_start, _b_end, b_level, b_text in blocks:
        if b_start < obj_end:
            continue
        is_self_check = any(sc in b_text for sc in _SELF_CHECK_MARKERS)
        is_section_break = obj_level is not None and b_level is not None and b_level <= obj_level
        if is_self_check or is_section_break:
            sec_end = min(sec_end, b_start)
            break

    return [it for it in _list_items(html[obj_end:sec_end]) if 0 < len(it) <= 400]


def _charge(throughput, n: int = 1) -> None:
    """Charge a throughput counter if one was supplied (no-op otherwise)."""
    if throughput is not None:
        try:
            throughput.charge(n)
        except Exception:
            pass


def extract_course_clos(config: dict, *, throughput=None) -> dict:
    """Extract CLOs from the course syllabus endpoint and Bloom-classify them.

    Returns {"clos": [str], "bloom_index": int, "bloom_level": str, "source": str}.
    On any failure returns an empty result with source="none".
    `throughput`, when supplied, is charged once for the syllabus fetch.
    """
    empty = {"clos": [], "bloom_index": 0, "bloom_level": "Unknown", "source": "none"}
    try:
        _charge(throughput)
        syllabus = canvas_api.get_syllabus(config)
    except Exception as exc:
        logger.warning("objective_extractor: syllabus fetch failed: %s", exc)
        return empty
    html = (syllabus or {}).get("html") or "" if isinstance(syllabus, dict) else (syllabus or "")
    clos = _outcomes_from_syllabus_html(html)
    if not clos:
        # CLOs may be in an attached docx/pdf (not parsed here).
        empty["source"] = "syllabus_no_outcomes_found"
        return empty
    idx, level = _classify(clos)
    return {"clos": clos, "bloom_index": idx, "bloom_level": level, "source": "syllabus"}


def _page_body(config: dict, slug: str, throughput=None) -> str:
    """Best-effort fetch of a wiki page body by slug. Empty string on failure."""
    try:
        _charge(throughput)
        page = canvas_api.get_page(config, slug)
    except Exception as exc:
        logger.debug("objective_extractor: page fetch failed for %s: %s", slug, exc)
        return ""
    if isinstance(page, dict):
        return page.get("body") or ""
    return page or ""


def _overview_item(items: list[dict]) -> dict | None:
    """Pick the overview Page item from a module's items.

    Prefers a Page whose title names an overview/objectives/start-here page;
    falls back to the first Page item.
    """
    pages = [it for it in items if it.get("type") == "Page"]
    for it in pages:
        title = (it.get("title") or "").lower()
        if any(w in title for w in ("overview", "objective", "start here", "welcome", "introduction")):
            return it
    return pages[0] if pages else None


def extract_module_mlos(config: dict, *, tree: list[dict] | None = None, throughput=None) -> dict:
    """Extract MLOs per module from module overview pages and Bloom-classify them.

    Returns {module_name: {"mlos": [str], "bloom_index": int, "bloom_level": str}}.
    Modules with no overview page or no MLO lines are omitted. Empty on failure.

    `tree`, when supplied, is reused (avoids a redundant module fetch -- the caller
    may already hold the tree). `throughput`, when supplied, is charged for the
    module fetch (if performed) and for each overview-page fetch.
    """
    if tree is None:
        try:
            _charge(throughput)
            tree = canvas_api.get_modules(config, include_items=True)
        except Exception as exc:
            logger.warning("objective_extractor: modules fetch failed: %s", exc)
            return {}

    result: dict[str, dict] = {}
    for mod in tree or []:
        name = mod.get("name") or ""
        items = mod.get("items") or []
        ov = _overview_item(items)
        if not ov:
            continue
        slug = ov.get("page_url") or ov.get("url") or ov.get("slug")
        if not slug:
            continue
        body = _page_body(config, slug, throughput)
        if not body:
            continue
        text = _block_text(body)
        mlos = [m.strip() for m in _MLO_RE.findall(text) if len(m.strip()) > 6]
        # Trim each MLO to its first sentence (drops trailing run-on into next MLO).
        mlos = [re.split(r"(?<=[.;])\s", m)[0].strip(" .;") for m in mlos]
        mlos = [m for m in mlos if m]
        if not mlos:
            # Fallback: objectives listed as plain <li> under a "Learning
            # Objectives" heading, with no "MLO N.N" prefix.
            mlos = [m.strip(" .;") for m in _mlos_from_heading_list(body)]
            mlos = [m for m in mlos if m]
        if not mlos:
            continue
        idx, level = _classify(mlos)
        result[name] = {"mlos": mlos, "bloom_index": idx, "bloom_level": level}
    return result


if __name__ == "__main__":
    import argparse
    import json

    parser = argparse.ArgumentParser(description="Extract CLOs/MLOs for a Canvas course.")
    parser.add_argument("--course-id", required=True)
    args = parser.parse_args()
    cfg = canvas_api.get_config(course_id=args.course_id)
    out = {
        "course_clos": extract_course_clos(cfg),
        "module_mlos": extract_module_mlos(cfg),
    }
    print(json.dumps(out, indent=2))
