#!/usr/bin/env  python
import datetime
import json
import re
from contextlib import closing

from mechanize import Request

from calibre.web.feeds.recipes import BasicNewsRecipe


class NRC(BasicNewsRecipe):
    title = 'NRC'
    __author__ = 'Cristi Ghera'
    max_articles_per_feed = 100
    description = 'NRC - Nieuws, achtergronden en onderzoeksjournalistiek'
    needs_subscription = False
    language = 'nl'
    country = 'NL'
    category = 'news, politics, Netherlands'
    resolve_internal_links = True
    remove_tags_before = {'class': 'article__header-and-content'}
    remove_tags_after = {'class': 'article__header-and-content'}
    remove_tags = [
        dict(
            attrs={
                'class': [
                    'article__footer',
                    'lees-ook',
                    'luister-naar',
                    'print-layout-warning',
                    'newslettersignup',
                    'article__byline',
                    'article__published-in',
                    'article__featured-image__caption__producer',
                    'metabox',
                ]
            }
        ),
        dict(name=['script', 'noscript', 'style']),
    ]
    remove_attributes = ['class', 'id', 'name', 'style']
    encoding = 'utf-8'
    no_stylesheets = True
    ignore_duplicate_articles = {'url'}
    delay = 0.3

    touchscreen = True

    frontpage = None

    title_regexp = None

    @staticmethod
    def _monthly_list_url(date, fmt='%Y/%m/'):
        return 'https://www.nrc.nl/de/data/NH/' + date.strftime(fmt)

    def _clean_article_title(self, title):
        if not title:
            return title
        if self.title_regexp is None:
            self.title_regexp = re.compile(r'<span class="keyword">([^<]+)</span>\s*')
        return self.title_regexp.sub(r'\1 ', title)

    def parse_index(self):
        sections = []
        today = datetime.date.today()
        headers = {
            'X-Requested-With': 'XMLHttpRequest',
            'Accept': 'application/json, text/javascript, */*; q=0.01',
            'DNT': '1',
        }
        monthly_list_urls = [self._monthly_list_url(today), self._monthly_list_url(datetime.date(today.year, today.month, 1) - datetime.timedelta(days=1))]
        issue_url = None
        issue_date = None
        for monthly_list_url in monthly_list_urls:
            with closing(self.browser.open(Request(monthly_list_url, None, headers))) as r:
                issues = json.loads(r.read())
                if len(issues) > 0:
                    issue_date = datetime.datetime.strptime(issues[0]['published_at'], '%Y-%m-%dT%H:%M:%SZ')
                    issue_url = self._monthly_list_url(issue_date, '%Y/%m/%d/')
                    self.frontpage = issues[0]['frontpage']
                    break
        if issue_url is None:
            return []
        with closing(self.browser.open(Request(issue_url, None, headers))) as r:
            edition = json.loads(r.read())
        documents = {}
        for headline in edition['paperheadlines']:
            item = headline['item']
            documents[headline['document_id']] = dict(url=item['full_url'], headline=self._clean_article_title(item['headline']))
        for section in edition['sections']:
            articles = []
            for doc in section['document_ids']:
                if doc not in documents:
                    self.log.warn('Document not found:', doc)
                    continue
                articles.append(dict(title=documents[doc]['headline'], url=documents[doc]['url']))
            sections.append((section['name'], articles))
        return sections

    def preprocess_html(self, soup):
        for tag in soup():
            if tag.name == 'img':
                if tag.has_attr('data-src-medium'):
                    tag['src'] = tag['data-src-medium'].split('|')[0]
                elif tag.has_attr('data-src'):
                    tag['src'] = tag['data-src'].split('|')[0]
                if tag['src'].startswith('//'):
                    tag['src'] = 'https:' + tag['src']
                elif tag['src'].startswith('/'):
                    tag['src'] = 'https://www.nrc.nl' + tag['src']
        if self.browser.cookiejar:
            self.browser.cookiejar.clear()
        return soup

    def get_cover_url(self):
        return self.frontpage
