#!/usr/bin/env python3
"""記事の本数・平均を再現する（標準ライブラリのみ、本文は出力しない）。

python3 -I article_counts.py --ai texts.jsonl --human human-clean-v1-18.jsonl --out results-v1/article-counts.md
公開CSVから: --human-csv per-text-human-full.csv --human-600-csv per-text-human-600.csv
--csv-dir は v1.1 の各本の数値だけを出力。人のIDは入力順にh01〜、対応表は出力しない。
"""
import argparse
from collections import Counter
import csv
from pathlib import Path
import re
import statistics as st
import sys

sys.path.insert(0, str(Path(__file__).resolve().parent))
from ai_style_count import (count_text, excerpt_at_sentence_end, load_jsonl,
                            markers_for, sentences, strip_ws)

TOPICS = {"s01", "s02", "s03", "s04", "s11"}
EXAMPLE_ID = "s02|openai/gpt-6.1-sol|plain|0"
# 旧shares.pyのinline検索。定型句の表には使わず、互換集計として明示する。
INLINE = {"いかがでしたか": r"いかがでしたか", "することができ": r"することができ",
          "と考えられ": r"と考えられ|と思われ", "と言えるでしょう": r"と言えるでしょう|と言えます",
          "結論から": r"結論から言うと|結論として", "重要です": r"(重要|大切|鍵)です", "ぜひ": r"ぜひ"}
FORMATS = {"見出し": "F01", "箇条書き": "F02", "太字": "F03", "区切り線": "F05a"}


def numeric_features(text):
    row = count_text(text, "v1.1")
    lens = [len(strip_ws(s)) for s in sentences(text)]
    row.update(sentence_count=len(lens),
               short_sentence_rate=sum(n <= 15 for n in lens) / max(1, len(lens)),
               long_sentence_rate=sum(n >= 60 for n in lens) / max(1, len(lens)))
    row.update({f"inline_{i}": int(bool(re.search(p, text))) for i, p in enumerate(INLINE.values(), 1)})
    return row


def human_rows(items, excerpt=False):
    return [{"id": f"h{i:02}", "questionId": item["questionId"], "source_type": item["condition"],
             **numeric_features(excerpt_at_sentence_end(item["text"], 600) if excerpt else item["text"])}
            for i, item in enumerate(items, 1)]


def read_human_csv(path):
    with open(path, encoding="utf-8", newline="") as f:
        return [{k: (v if k in {"id", "questionId", "source_type"} else float(v))
                 for k, v in row.items()} for row in csv.DictReader(f)]


def write_csv(path, rows):
    with Path(path).open("w", encoding="utf-8", newline="") as f:
        writer = csv.DictWriter(f, fieldnames=list(rows[0]))
        writer.writeheader()
        writer.writerows(rows)


def table(headers, rows):
    return ["| " + " | ".join(headers) + " |", "|" + "---|" * len(headers),
            *("| " + " | ".join(map(str, row)) + " |" for row in rows), ""]


def build_report(ai_items, ai, full, excerpt):
    out = ["# 記事用の本数・平均", "",
           "定型句表はv1のMARKERSを使い、1回以上当たる文章を1本とする。H02も併記。"
           "書式・文長はv1.1。書式いずれか＝F01・F02・F03・F05aの論理和。"
           "平均は各本の値（count_textで丸めた値）を等重みで平均。文を全本で合算した平均ではない。"
           "短文率は15字以下、長文率は60字以上の文の割合を各本で出して平均。空白は除く。", ""]
    marker_rows = []
    for mid, _, desc, _, _ in markers_for("v1"):
        if mid.startswith("P") or mid == "H02":
            presence = [(item["condition"], count_text(item["text"], "v1")[mid] > 0) for item in ai_items]
            marker_rows.append((mid, desc, sum(v for _, v in presence),
                                sum(v for c, v in presence if c == "plain")))
    out += ["## 定型句の表（v1）", ""] + table(["ID", "規則", "全360本", "plain180本"], marker_rows)
    groups = []
    for model in sorted({r["model"] for r in ai}):
        short = model.split("/")[-1]
        for condition in ("plain", "natural"):
            groups.append((f"12お題 {short} {condition}", [r for r in ai if r["model"] == model and r["condition"] == condition]))
        groups.append((f"12お題 {short} 2条件", [r for r in ai if r["model"] == model]))
        for condition in ("plain", "natural", "2条件"):
            groups.append((f"5お題 {short} {condition}", [r for r in ai if r["model"] == model and r["questionId"] in TOPICS
                                                           and (condition == "2条件" or r["condition"] == condition)]))
    groups += [("人18本 全文", full), ("人18本 約600字", excerpt)]
    out += ["## 書式を1回以上含む本数（v1.1）", ""]
    out += table(["群", "本数", *FORMATS, "書式いずれか"],
                 [(name, len(rows), *(sum(r[k] > 0 for r in rows) for k in FORMATS.values()),
                   sum(any(r[k] > 0 for k in FORMATS.values()) for r in rows)) for name, rows in groups])
    out += ["## 旧shares.pyのinline検索の本数（定型句表とは別定義）", ""]
    out += table(["群", "本数", *INLINE], [(name, len(rows), *(int(sum(r[f"inline_{i}"] for r in rows))
                      for i in range(1, len(INLINE) + 1))) for name, rows in groups])
    metrics = ("S01_mean_sent_len", "S05_cv_sent_len", "S06_adj_len_change", "R05_neg_then_pos",
               "chars", "short_sentence_rate", "long_sentence_rate")
    out += ["## 平均と字数の範囲（v1.1）", ""]
    out += table(["群", "本数", *metrics, "最小字数", "最大字数", "S05最小"],
                 [(name, len(rows), *(f"{st.mean(r[k] for r in rows):.6f}" for k in metrics),
                   int(min(r['chars'] for r in rows)), int(max(r['chars'] for r in rows)),
                   f"{min(r['S05_cv_sent_len'] for r in rows):.3f}") for name, rows in groups])
    out += ["## 人の出どころの種類とお題の本数", ""]
    out += table(["種類", "本数"], sorted(Counter(r["source_type"] for r in full).items()))
    out += table(["お題", "本数"], sorted(Counter(r["questionId"] for r in full).items()))
    out += ["## 生成のメタデータ", "",
            f"本数 {len(ai_items)}。finishReason: {dict(sorted(Counter(r['finishReason'] for r in ai_items).items()))}。",
            f"生成日時 {min(r['at'] for r in ai_items)} 〜 {max(r['at'] for r in ai_items)}。",
            f"API実費合計 ${sum(float(r['costUsd'] or 0) for r in ai_items):.6f}。", "",
            "## 図の実例と空白の扱い", "", f"対象: {EXAMPLE_ID}。"]
    example = next(r for r in ai_items if r["id"] == EXAMPLE_ID)
    raw = [len(s) for s in sentences(example["text"])]
    clean = [len(strip_ws(s)) for s in sentences(example["text"])]
    out += table(["文長の数え方", "文数", "15〜40字の文数", "平均字数", "変動係数"],
                 [(label, len(lens), sum(15 <= n <= 40 for n in lens), f"{st.mean(lens):.6f}",
                   f"{st.pstdev(lens)/st.mean(lens):.6f}")
                  for label, lens in (("従来の図 len(s)", raw), ("count_textと同じ 空白除去", clean))])
    changed = sum(any(len(s) != len(strip_ws(s)) for s in sentences(item["text"])) for item in ai_items)
    out += [f"定義は異なる。文内空白で文長が変わるAIは全360本中{changed}本。"
            f"実例の文長の一致: {raw == clean}。make_figures.pyは空白を除く定義に統一した。", ""]
    return "\n".join(out)


def main():
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("--ai", required=True)
    source = ap.add_mutually_exclusive_group(required=True)
    source.add_argument("--human")
    source.add_argument("--human-csv")
    ap.add_argument("--human-600-csv")
    ap.add_argument("--csv-dir", type=Path)
    ap.add_argument("--out", type=Path, required=True)
    args = ap.parse_args()
    items = load_jsonl(args.ai)
    ai = [{**{k: item[k] for k in ("id", "questionId", "model", "condition", "runIndex")},
           **numeric_features(item["text"])} for item in items]
    if args.human:
        humans = load_jsonl(args.human)
        full, excerpt = human_rows(humans), human_rows(humans, True)
    else:
        if not args.human_600_csv:
            ap.error("--human-csvには--human-600-csvも必要です")
        full, excerpt = read_human_csv(args.human_csv), read_human_csv(args.human_600_csv)
    for rows in (full, excerpt):
        if len(rows) != 18 or Counter(r['questionId'] for r in rows) != Counter(s01=4, s02=4, s03=3, s04=3, s11=4):
            ap.error("人は本文化した18本（s01:4,s02:4,s03:3,s04:3,s11:4）が必要です")
    if len(ai) != 360:
        ap.error("AIは360本が必要です")
    args.out.parent.mkdir(parents=True, exist_ok=True)
    args.out.write_text(build_report(items, ai, full, excerpt), encoding="utf-8")
    if args.csv_dir:
        args.csv_dir.mkdir(parents=True, exist_ok=True)
        for name, rows in (("ai-360", ai), ("human-full", full), ("human-600", excerpt)):
            write_csv(args.csv_dir / f"per-text-{name}.csv", rows)
    print(f"{args.out}: AI {len(ai)}本、人 {len(full)}本（本文・出典URLは出力しません）")


if __name__ == "__main__":
    main()