#!/usr/bin/env  python
import json
import time
from datetime import datetime, timedelta
from urllib.parse import quote, urlencode
from uuid import uuid4

from html5_parser import parse
from lxml import etree
from mechanize import Request

from calibre import browser
from calibre.utils.date import local_tz
from calibre.web.feeds.news import BasicNewsRecipe


def safe_dict(data, *names):
    ans = data
    for x in names:
        ans = ans.get(x) or {}
    return ans


def process_web_list(li_node):
    li_html = ''
    for li in li_node['items']:
        if li.get('textHtml'):
            li_html += f'<li>{li["textHtml"]}</li>'
        elif li.get('textJson'):
            li_html += f'<li>{parse_textjson(li["textJson"])}</li>'
        else:
            li_html += f'<li>{li.get("text", "")}</li>'
    return '<ul>' + li_html + '</ul>'


def process_info_box(bx):
    info = ''
    for x in safe_dict(bx, 'components'):
        info += f'<blockquote>{process_web_node(x)}</blockquote>'
    return info


def parse_txt(ty):
    typ = ty.get('type', '')
    children = ty.get('children', [])
    href = '#'
    attributes = ty.get('attributes') or ()
    for a in attributes:
        if a.get('name') == 'href':
            href = a.get('value', href)
            break

    tag_map = {
        'text': lambda: [ty.get('value', '')],
        'scaps': lambda: [
            f'<span style="text-transform: uppercase; font-size: 0.85em; letter-spacing: 0.05em;">{"".join(parse_txt(c))}</span>' for c in children
        ],
        'bold': lambda: [f'<b>{"".join(parse_txt(c))}</b>' for c in children],
        'drop_caps': lambda: [f'<b>{"".join(parse_txt(c))}</b>' for c in children],
        'italic': lambda: [f'<i>{"".join(parse_txt(c))}</i>' for c in children],
        'linebreak': lambda: ['<br>'],
        'external_link': lambda: [f'<a href="{href}">{"".join(parse_txt(c))}</a>' for c in children],
        'internal_link': lambda: [f'<a href="{href}">{"".join(parse_txt(c))}</a>' for c in children],
        'ufinish': lambda: [text for c in children for text in parse_txt(c)],
        'subscript': lambda: [f'<sub>{"".join(parse_txt(c))}</sub>' for c in children],
        'superscript': lambda: [f'<sup>{"".join(parse_txt(c))}</sup>' for c in children],
    }

    if typ in tag_map:
        yield from tag_map[typ]()
    else:
        print('** ', typ)


def parse_textjson(nt):
    return ''.join(''.join(parse_txt(n)) for n in nt)


def process_web_node(node):
    ntype = node.get('type', '')
    if ntype == 'CROSSHEAD':
        if node.get('textHtml'):
            return f'<h4>{node.get("textHtml")}</h4>'
        return f'<h4>{node.get("text", "")}</h4>'
    elif ntype in ['PARAGRAPH', 'BOOK_INFO']:
        if node.get('textHtml'):
            return f'\n<p>{node.get("textHtml")}</p>'
        elif node.get('textJson'):
            return f'\n<p>{parse_textjson(node["textJson"])}</p>'
        return f'\n<p>{node.get("text", "")}</p>'
    elif (ntype == 'IMAGE') or (node.get('__typename', '') == 'ImageComponent'):
        alt = '' if node.get('altText') is None else node.get('altText')
        cap = ''
        if node.get('caption'):
            if node['caption'].get('textHtml') is not None:
                cap = node['caption']['textHtml']
            elif node['caption'].get('textJson') is not None:
                cap = parse_textjson(node['caption']['textJson'])
            elif node['caption'].get('text') is not None:
                cap = node['caption']['text']
        return f'<div><img src="{node["url"]}" title="{alt}"></div><div style="text-align:center; font-size:small;">{cap}</div>'
    elif ntype == 'PULL_QUOTE':
        if node.get('textHtml'):
            return f'<blockquote>{node.get("textHtml")}</blockquote>'
        elif node.get('textJson'):
            return f'<blockquote>{parse_textjson(node["textJson"])}</blockquote>'
        return f'<blockquote>{node.get("text", "")}</blockquote>'
    elif ntype == 'BLOCK_QUOTE':
        if node.get('textHtml'):
            return f'<blockquote><i>{node.get("textHtml")}</i></blockquote>'
        elif node.get('textJson'):
            return f'<blockquote><i>{parse_textjson(node["textJson"])}</i></blockquote>'
        return f'<blockquote><i>{node.get("text", "")}</i></blockquote>'
    elif ntype == 'DIVIDER':
        return '<hr>'
    elif ntype == 'INFOGRAPHIC':
        if node.get('fallback'):
            return process_web_node(node['fallback'])
    elif ntype == 'INFOBOX':
        return process_info_box(node)
    elif ntype == 'UNORDERED_LIST':
        if node.get('items'):
            return process_web_list(node)
    elif ntype:
        print('** ', ntype)
    return ''


def load_article_from_json(raw):
    # open('/t/raw.json', 'w').write(raw)
    body = ''
    try:
        data = json.loads(raw)['data']['findArticleByUrl']
    except Exception:
        try:
            data = json.loads(raw)['props']['pageProps']['cp2Content']
        except Exception:
            data = json.loads(raw)['props']['pageProps']['content']

    body += f'<div style="color: red; font-size:small; font-weight:bold;">{data.get("flyTitle", "")}</div>'
    body += f'<h1>{data["headline"]}</h1>'
    if data.get('rubric') and data.get('rubric') is not None:
        body += f'<div style="font-style: italic; color:#202020;">{data.get("rubric", "")}</div>'
    try:
        date = data['dateModified']
    except Exception:
        date = data['datePublished']
    dt = datetime.fromisoformat(date[:-1]) + timedelta(seconds=time.timezone)
    dt = dt.strftime('%b %d, %Y %I:%M %p')
    if data.get('dateline') is None:
        body += f'<p style="color: gray; font-size: small;">{dt}</p>'
    else:
        body += f'<p style="color: gray; font-size: small;">{dt + " | " + (data["dateline"])}</p>'
    main_image_url = safe_dict(data, 'leadComponent') or ''
    if main_image_url:
        body += process_web_node(data['leadComponent'])
    if data.get('byline'):
        if data['byline'] is not None:
            body += f'<p style="color: gray; font-size: small;"><i>{"By " + data["byline"]}</i></p>'
    for node in data.get('body'):
        body += process_web_node(node)
    return '<html><body><article>' + body + '</article></body></html>'


class NoArticles(Exception):
    pass


def get_content(url):
    headers = {
        'user-agent': 'Mozilla/5.0 (Linux; Android 16) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/150.0.7871.124 Mobile Safari/537.36 Liskov',
        'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
        'x-requested-with': 'com.economist.lamarr',
        'blaize_session': str(uuid4()),
        'sec-ch-ua-mobile': '?1',
        'sec-ch-ua-platform': 'Android',
        'upgrade-insecure-requests': '1',
    }
    br = browser()
    req = Request(
        url + '?app=core%2Cfocused&no-chrome=key&no-adverts=key&webview=liskov',
        headers=headers,
    )
    res = br.open(req)
    return res.read()


def process_url(url):
    if url.startswith('/'):
        url = 'https://www.economist.com' + url
    return url


def get_edition(issue_date=None):
    if issue_date:
        operation_name = 'EditionByDateQuery'
        operation_args = '($issueDate: Date!)'
        resolver = 'findEditionByDate(issueDate: $issueDate, editionType: WEEKLY)'
        variables = {'issueDate': issue_date}
    else:
        operation_name = 'LatestEditionQuery'
        operation_args = ''
        resolver = 'findLatestEdition(editionType: [WEEKLY])'
        variables = {}
    graphql = (
        'query ' + operation_name + operation_args + ' {'
        ' edition: ' + resolver + ' {'
        ' id issueDate datePublished url headline flyTitle rubric'
        ' cover { url }'
        ' sections { id name articles { id url headline flyTitle rubric } }'
        ' }'
        '}'
    )
    query = {
        'operationName': operation_name,
        'variables': json.dumps(variables, separators=(',', ':')),
        'query': graphql,
    }
    url = 'https://cp2-graphql-gateway.p.aws.economist.com/graphql?' + urlencode(query, safe='()!', quote_via=quote)
    return safe_dict(json.loads(get_content(url)), 'data', 'edition')


class Economist(BasicNewsRecipe):
    title = 'The Economist'
    language = 'en_GB'
    encoding = 'utf-8'
    masthead_url = 'https://www.livemint.com/lm-img/dev/economist-logo-oneline.png'

    __author__ = 'Kovid Goyal, unkn0wn'
    description = 'Global news and current affairs from a European perspective. Best downloaded away from Friday to avoid being blocked.'
    extra_css = '''
        em { color:#202020; }
        img {display:block; margin:0 auto;}
    '''

    resolve_internal_links = True

    delay = 2
    # browser_type = 'webengine'
    recipe_specific_options = {
        'date': {
            'short': 'The date of the edition to download (YYYY-MM-DD format)',
            'long': 'For example, 2024-07-19',
        },
        'res': {
            'short': 'For hi-res images, select a resolution from the\nfollowing options: 834, 960, 1096, 1280, 1424',
            'long': 'This is useful for non e-ink devices, and for a lower file size\nthan the default, use from 480, 384, 360, 256.',
            'default': '600',
        },
    }

    def get_browser(self, *args, **kwargs):
        # from calibre.utils.random_ua import common_english_word_ua
        #
        # self.last_used_ua = kwargs['user_agent'] = kwargs.get('user_agent', common_english_word_ua())

        self.last_used_ua = kwargs['user_agent'] = (
            'Mozilla/5.0 (Linux; Android 14) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.6533.103 Mobile Safari/537.36 Liskov'
        )
        br = BasicNewsRecipe.get_browser(self, *args, **kwargs)
        return br

    def canonicalize_internal_url(self, url, is_link=True):
        if url.endswith('/print'):
            url = url.rpartition('/')[0]
        return BasicNewsRecipe.canonicalize_internal_url(self, url, is_link=is_link)

    def publication_date(self):
        edition_date = self.recipe_specific_options.get('date')
        if isinstance(edition_date, str):
            edition_date = edition_date.strip()
        if edition_date and isinstance(edition_date, str):
            return datetime.strptime(edition_date, '%Y-%m-%d').replace(tzinfo=local_tz)
        try:
            url = self.browser.open('https://www.economist.com/printedition').geturl()
            date_str = url.split('/')[-1]
            return datetime.strptime(date_str, '%Y-%m-%d').replace(tzinfo=local_tz)
        except Exception as e:
            self.log('Failed to fetch publication date from website with error: ' + str(e))
        try:
            edition = self.economist_get_edition()
            return datetime.strptime(edition['issueDate'][:10], '%Y-%m-%d').replace(tzinfo=local_tz)
        except Exception as e:
            self.log('Failed to fetch publication date from GraphQL with error: ' + str(e))
            return super().publication_date()

    def economist_get_edition(self, edition_date=None):
        cache = getattr(self, '_edition_cache', {})
        if edition_date not in cache:
            cache[edition_date] = get_edition(edition_date)
            self._edition_cache = cache
        return cache[edition_date]

    def get_obfuscated_article(self, url):
        return {'data': get_content(url), 'url': url}

    def economist_return_index(self, ans):
        if not ans:
            raise NoArticles(
                'Could not find any articles, either the '
                'economist.com server is having trouble and you should '
                'try later or the website format has changed and the '
                'recipe needs to be updated.'
            )
        return ans

    def economist_test_article(self):
        return [('Articles', [{'title': 'test', 'url': 'https://www.economist.com/international/2026/07/23/the-geopolitics-of-air-conditioning'}])]

    def parse_index(self):
        # return self.economist_test_article()
        edition_date = self.recipe_specific_options.get('date')
        if isinstance(edition_date, str):
            edition_date = edition_date.strip()
        if edition_date and isinstance(edition_date, str):
            url = 'https://www.economist.com/weeklyedition/' + edition_date
            self.timefmt = ' [' + edition_date + ']'
        else:
            edition_date = None
            url = 'https://www.economist.com/weeklyedition'
        try:
            ans = self.economist_parse_index(url)
        except Exception as e:
            self.log('Failed to get index from website with error:', e)
            ans = []
        if not ans:
            self.log('Falling back to GraphQL for the edition index')
            try:
                ans = self.economist_parse_graphql_index(self.economist_get_edition(edition_date))
            except Exception as e:
                self.log('Failed to get index from GraphQL with error:', e)
                ans = []
        return self.economist_return_index(ans)

    def economist_parse_index(self, url):
        soup = self.index_to_soup(get_content(url))
        script_tag = soup.find('script', id='__NEXT_DATA__')
        if script_tag is None:
            self.log('Failed to get index with UA:', self.last_used_ua)
            from calibre.utils.random_ua import random_common_chrome_user_agent

            user_agent = random_common_chrome_user_agent()
            self.log('Retrying with UA:', user_agent)
            time.sleep(1)
            self.browser = self.get_browser(user_agent=user_agent)
            soup = self.index_to_soup(url)
            script_tag = soup.find('script', id='__NEXT_DATA__')
        if script_tag is None:
            return []
        data = json.loads(script_tag.string)
        # open('/t/raw.json', 'w').write(json.dumps(data, indent=2, sort_keys=True))

        self.description = safe_dict(data, 'props', 'pageProps', 'content', 'headline')
        self.timefmt = ' [' + safe_dict(data, 'props', 'pageProps', 'content', 'formattedIssueDate') + ']'
        self.cover_url = (
            safe_dict(data, 'props', 'pageProps', 'content', 'cover', 'url')
            .replace(
                'economist.com/',
                'economist.com/cdn-cgi/image/width=960,quality=80,format=auto/',
            )
            .replace('SQ_', '')
            .replace('_AS', '_AP')
        )
        self.log('Got cover:', self.cover_url)

        feeds = []
        seen = set()
        for component in safe_dict(data, 'props', 'pageProps', 'content', 'components'):
            match component.get('type'):
                case 'AD':
                    continue
                case 'COLLECTION':
                    articles = []
                    section = component['name']
                    self.log(section)
                    for article in component['articles']:
                        for article in component['articles']:
                            if article['id'] in seen:
                                continue
                            seen.add(article['id'])
                            title = article['headline']
                            url = process_url(article['url'])
                            sub = article.get('flyTitle') or ''
                            desc = article.get('rubric') or ''
                            if sub and section != sub:
                                desc = sub + ' :: ' + desc
                            self.log('\t', title, '\n\t', desc, '\n\t\t', url)
                            articles.append({'title': title, 'url': url, 'description': desc})
                    if articles:
                        feeds.append((section, articles))

        return feeds

    def economist_parse_graphql_index(self, edition):
        if not edition:
            return []
        self.description = edition.get('headline') or self.description
        self.timefmt = ' [' + edition['issueDate'][:10] + ']'
        cover_url = safe_dict(edition, 'cover', 'url')
        if cover_url:
            self.cover_url = (
                cover_url
                .replace(
                    'economist.com/',
                    'economist.com/cdn-cgi/image/width=960,quality=80,format=auto/',
                )
                .replace('SQ_', '')
                .replace('_AS', '_AP')
            )
            self.log('Got cover:', self.cover_url)

        feeds = []
        seen = set()
        for component in edition.get('sections') or ():
            articles = []
            section = component.get('name') or ''
            self.log(section)
            for article in component.get('articles') or ():
                url = process_url(article.get('url') or '')
                article_id = article.get('id') or url
                title = article.get('headline') or ''
                if not article_id or article_id in seen or not title or not url:
                    continue
                seen.add(article_id)
                sub = article.get('flyTitle') or ''
                desc = article.get('rubric') or ''
                if sub and section != sub:
                    desc = sub + (' :: ' + desc if desc else '')
                self.log('\t', title, '\n\t', desc, '\n\t\t', url)
                articles.append({'title': title, 'url': url, 'description': desc})
            if articles:
                feeds.append((section, articles))

        return feeds

    def preprocess_html(self, soup):
        width = '600'
        w = self.recipe_specific_options.get('res')
        if w and isinstance(w, str):
            width = w
        for img in soup.findAll('img', src=True):
            qua = 'economist.com/cdn-cgi/image/width=' + width + ',quality=80,format=auto/'
            img['src'] = img['src'].replace('economist.com/', qua)
        return soup

    def preprocess_raw_html(self, raw_html, url):
        # open('/t/raw.html', 'wb').write(raw.encode('utf-8'))
        if '/interactive/' in url:
            return '<html><body><article><h1>Article not available.</h1><em>This article is supposed to be read in a browser.</em></article></body></html>'
        root_ = parse(raw_html)
        script = root_.xpath('//script[@id="__NEXT_DATA__"]')
        html = load_article_from_json(script[0].text)
        root = parse(html)

        for x in root.xpath('//*[name()="script" or name()="style" or name()="source" or name()="meta"]'):
            x.getparent().remove(x)
        # the economist uses <small> for small caps with a custom font
        for init in root.xpath('//span[@data-caps="initial"]'):
            init.set('style', 'font-weight:bold;')
        for x in root.xpath('//small'):
            if x.text and len(x) == 0:
                x.text = x.text.upper()
                x.tag = 'span'
                x.set(
                    'style',
                    'text-transform: uppercase; font-size: 0.85em; letter-spacing: 0.05em;',
                )
        for h2 in root.xpath('//h2'):
            h2.tag = 'h4'
        for x in root.xpath('//figcaption'):
            x.set('style', 'text-align:center; font-size:small;')
        for x in root.xpath('//cite'):
            x.tag = 'blockquote'
            x.set('style', 'color:#404040;')
        raw = etree.tostring(root, encoding='unicode')
        return raw
