#!/usr/bin/env python3
"""保存済み HTML の本文を、集計用の Markdown に近いテキストへ変換する。"""

import argparse
import re
from html.parser import HTMLParser
from pathlib import Path


EXCLUDED_TAGS = {"script", "style", "nav", "header", "footer", "aside", "form", "noscript", "iframe"}
# 「ad」は部分一致にすると header・head・loaded などに当たって本文ごと消えるので、語の単位でだけ見る
EXCLUDED_NAME = re.compile(
    r"comment|sidebar|share|related|breadcrumb|author|profile"
    r"|(?:^|[\s_-])(?:ads?|advert\w*|adsense|adslot|ad\d*)(?:[\s_-]|$)",
    re.I,
)
VOID_TAGS = {"area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta", "param", "source", "track", "wbr"}


class _Node:
    def __init__(self, tag="", attrs=()):
        self.tag = tag
        self.attrs = dict(attrs)
        self.children = []


class _TreeBuilder(HTMLParser):
    def __init__(self):
        super().__init__(convert_charrefs=True)
        self.root = _Node()
        self.stack = [self.root]

    def handle_starttag(self, tag, attrs):
        node = _Node(tag, attrs)
        self.stack[-1].children.append(node)
        if tag not in VOID_TAGS:
            self.stack.append(node)

    def handle_startendtag(self, tag, attrs):
        self.handle_starttag(tag, attrs)
        if tag not in VOID_TAGS:
            self.handle_endtag(tag)

    def handle_endtag(self, tag):
        for i in range(len(self.stack) - 1, 0, -1):
            if self.stack[i].tag == tag:
                del self.stack[i:]
                return

    def handle_data(self, data):
        self.stack[-1].children.append(data)

    def handle_entityref(self, name):
        # script/style 内では HTMLParser が実体参照を自動変換しない。
        self.handle_data(f"&{name};")

    def handle_charref(self, name):
        self.handle_data(f"&#{name};")


def _excluded(node):
    if node.tag in EXCLUDED_TAGS:
        return True
    # 本文を入れる枠の要素は名前で除かない（WordPress は body に sidebar-right・author-xxx などを付けるため、
    # 名前で除くと本文の選択ごと空になる）
    if node.tag in ("html", "body", "main", "article"):
        return False
    name = " ".join(str(node.attrs.get(k) or "") for k in ("class", "id"))
    return bool(EXCLUDED_NAME.search(name) or re.search(r"\bwm-ipp\b", name, re.I))


def _first(node, tag):
    if isinstance(node, str) or _excluded(node):
        return None
    if node.tag == tag:
        return node
    for child in node.children:
        found = _first(child, tag)
        if found is not None:
            return found
    return None


def _render(node):
    if isinstance(node, str):
        return re.sub(r"\s+", " ", node)
    if _excluded(node):
        return ""
    tag = node.tag
    if tag == "br":
        return "\n"
    if tag in ("ul", "ol"):
        parts = []
        start = node.attrs.get("start", "1")
        try:
            number = int(start)
        except (TypeError, ValueError):
            number = 1
        for child in node.children:
            if isinstance(child, _Node) and child.tag == "li":
                marker = "- " if tag == "ul" else f"{number}. "
                parts.append(_render_list_item(child, marker))
                number += 1
            else:
                parts.append(_render(child))
        return "\n" + "".join(parts) + "\n"
    if tag == "li":
        return _render_list_item(node, "- ")
    content = "".join(_render(child) for child in node.children)
    if tag in ("strong", "b"):
        return f"**{content.strip()}**" if content.strip() else ""
    if re.fullmatch(r"h[1-6]", tag):
        return f"\n{'#' * int(tag[1])} {content.strip()}\n" if content.strip() else ""
    if tag in ("p", "div", "tr"):
        return "\n" + content + "\n"
    if tag in ("td", "th"):
        return content + " "
    return content


def _render_list_item(node, marker):
    if _excluded(node):
        return ""
    return "\n" + marker + "".join(_render(child) for child in node.children).strip() + "\n"


def html_to_markdownish(html: str) -> str:
    """article、main、body の順で本文を選び、HTML の書式を集計用に残す。"""
    parser = _TreeBuilder()
    parser.feed(html)
    parser.close()
    scope = parser.root
    for tag in ("article", "main", "body"):
        found = _first(parser.root, tag)
        if found is not None:
            scope = found
            break
    rendered = _render(scope)
    lines = [re.sub(r"[ \t]+", " ", line).strip() for line in rendered.splitlines()]
    return re.sub(r"\n{3,}", "\n\n", "\n".join(lines).strip())


def slice_by_heading(text: str, start: str, end: str | None = None) -> str:
    """開始見出しを除き、次の同レベル以上の見出し（指定時は終了見出し）まで返す。"""
    if not start:
        raise ValueError("開始見出しを指定してください")
    lines = text.splitlines()
    start_index = None
    start_level = None
    for i, line in enumerate(lines):
        heading = re.match(r"^(#+)[ \t]*(.*)$", line)
        if heading is None:
            continue
        level, title = len(heading[1]), heading[2]
        if start_index is None:
            if start in title:
                start_index, start_level = i + 1, level
        elif (end is not None and end in title) or (end is None and level <= start_level):
            return "\n".join(lines[start_index:i]).strip()
    if start_index is None:
        raise ValueError(f"開始見出しが見つかりません: {start}")
    if end is not None:
        raise ValueError(f"終了見出しが見つかりません: {end}")
    return "\n".join(lines[start_index:]).strip()


def read_html(path):
    """HTML の meta charset を読み、指定がなければ UTF-8 として読む。"""
    raw = Path(path).read_bytes()
    match = re.search(rb"<meta\b[^>]*\bcharset\s*=\s*['\"]?([\w.-]+)", raw[:8192], re.I)
    encoding = match.group(1).decode("ascii") if match else "utf-8-sig"
    try:
        return raw.decode(encoding, errors="replace")
    except LookupError:
        return raw.decode("utf-8-sig", errors="replace")


def main():
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("input", help="保存済み HTML ファイル")
    ap.add_argument("--out", help="出力先。省略時は標準出力")
    ap.add_argument("--section-start", help="抽出を開始する見出しの一部（見出し自体は含めない）")
    ap.add_argument("--section-end", help="抽出を終了する後続見出しの一部（省略時は次の同レベル以上）")
    args = ap.parse_args()
    if args.section_end is not None and args.section_start is None:
        ap.error("--section-end には --section-start が必要です")
    result = html_to_markdownish(read_html(args.input))
    if args.section_start is not None:
        try:
            result = slice_by_heading(result, args.section_start, args.section_end)
        except ValueError as exc:
            ap.error(str(exc))
    result += "\n"
    if args.out:
        Path(args.out).write_text(result, encoding="utf-8")
    else:
        print(result, end="")


if __name__ == "__main__":
    main()