#!/usr/bin/env python

import re
from collections import defaultdict
from datetime import date, timedelta

from calibre.web.feeds.news import BasicNewsRecipe

DIR_COLLECTIONS = [
    ['world'],
    ['nation'],
    ['politics'],
    ['opinion', 'op-ed', 'opinion-la', 'editorials', 'readersreact', 'topoftheticket', 'endorsements'],
    ['local', 'lanow', 'california', 'crime', 'abcarian', 'education', 'weather'],
    ['business', 'hollywood', 'technology'],
    ['sports'],
    ['entertainment', 'movies', 'music', 'tv', 'arts', 'gossip', 'envelope'],
    ['books'],
    ['food', 'jonathon-gold', 'dailydish'],
    ['health'],
    ['style', 'laaffairs', 'pets'],
    ['science', 'sciencenow'],
    ['home'],
    ['travel'],
    ['fashion'],
]


def classes(classes):
    q = frozenset(classes.split(' '))
    return dict(attrs={'class': lambda x: x and frozenset(x.split()).intersection(q)})


def absurl(url):
    if url.startswith('/'):
        url = 'https://www.latimes.com' + url
    return url


def what_section(url):
    parts = url.split('/')
    return parts[-4].capitalize()


class LATimes(BasicNewsRecipe):
    title = 'Los Angeles Times'
    __author__ = 'Jose Ortiz'
    description = 'The Los Angeles Times is a leading source of news on Southern California, entertainment, movies, television, music, politics, business, health, technology, travel, sports, environment, economics, autos, jobs, real estate and other topics affecting California'  # noqa: E501
    category = 'news, politics, USA, Los Angeles, world'
    oldest_article = 1
    max_articles_per_feed = 200
    no_stylesheets = True
    encoding = 'utf8'
    use_embedded_content = False
    compress_news_images = True
    compress_news_images_auto_size = 5
    language = 'en_US'
    remove_empty_feeds = True
    ignore_duplicate_articles = {'url'}
    publication_type = 'newspaper'
    cover_url = 'http://www.latimes.com/includes/sectionfronts/A1.pdf'

    keep_only_tags = [
        classes('headline page-lead-media authors published-date page-article-container'),
        dict(attrs={'data-element': 'story-body'}),
    ]

    remove_tags = [classes('google-dfp-ad-wrapper enhancement')]

    def parse_index(self):
        index = 'https://www.latimes.com/'
        # Constrain the date path-segment to today+yesterday so the section
        # indexes don't pull in arbitrary older articles. Same idea as
        # chicago_tribune.recipe's inline tdy/yest filter (just expressed as
        # a regex alternation since latimes uses re.compile).
        tdy = date.today().strftime('%Y-%m-%d')
        yest = (date.today() - timedelta(days=1)).strftime('%Y-%m-%d')
        pat = r'^https://www\.latimes\.com/[^/]+?/story/(' + tdy + '|' + yest + r')/\S+'
        articles = self.find_articles(index, pat)
        for collection in DIR_COLLECTIONS:
            if self.test:
                continue
            topdir = collection.pop(0)
            collection_index = index + topdir + '/'
            articles += self.find_articles(collection_index, pat)
            for subdir in collection:
                sub_index = collection_index + subdir + '/'
                articles += self.find_articles(sub_index, pat)

        feeds = defaultdict(list)
        for article in articles:
            section = what_section(article['url'])
            feeds[section].append(article)

        return [(k, feeds[k]) for k in sorted(feeds)]

    def preprocess_html(self, soup):
        for img in soup.findAll('img', attrs={'data-src': True}):
            if img.findParent('a', href='http://www.latimes.com/opinion/la-letter-to-the-editor-htmlstory.html') is img.parent and img['data-src'].endswith(
                '/la-letter-to-the-editor'
            ):
                img.parent.extract()
            else:
                img['src'] = img['data-src']
        # Strip <picture> wrappers and viewport sizing hints. <source srcset>
        # URLs are remote and never load in an EPUB (no network at read
        # time), and sizes="100vw" on the <img> tells the reader to render
        # at full viewport width — combined with the image's natural aspect
        # ratio that overflows pages for portrait photos. Drop <source>,
        # unwrap <picture> so the <img> rises to its parent, and remove the
        # sizes/fetchpriority hints so the reader respects the image's
        # embedded dimensions.
        for src in soup.findAll('source'):
            src.decompose()
        for picture in soup.findAll('picture'):
            picture.unwrap()
        for img in soup.findAll('img'):
            for attr in ('sizes', 'fetchpriority'):
                if attr in img.attrs:
                    del img[attr]
        # Drop the Twitter "Follow" link from each byline. The anchor wraps
        # an SVG <use> sprite reference that points at a <shape> in a
        # sibling <svg> elsewhere in the source document; that cross-
        # document reference can't resolve in the converted EPUB, so the
        # anchor renders as an empty icon slot followed by the orphan word
        # "Follow" next to the byline. The "Staff Writer" <span> sibling
        # is not a descendant of the anchor, so the rubric stays in place.
        for a in soup.findAll('a', attrs={'data-social-trigger': 'enhancedByline'}):
            a.decompose()
        return soup

    def find_articles(self, index, pattern):
        self.log('Downloading and parsing index: ', index)
        self.log('Pattern: ', pattern)
        try:
            soup = self.index_to_soup(index)
        except Exception:
            self.log('Failed to download ', index)
            return []
        if soup.main is not None:
            alinks = soup.main.findAll('a', {'href': re.compile(pattern)})
        else:
            alinks = soup.findAll('a', {'href': re.compile(pattern)})
        alinks = [a for a in alinks if len(a.contents) == 1 and a.find(text=True, recursive=False)]
        articles = [{'title': self.tag_to_string(a), 'url': absurl(a['href'])} for a in alinks]
        # Populate `description` from each article page's
        cache = getattr(self, '_og_desc_cache', None)
        if cache is None:
            cache = self._og_desc_cache = {}
        for art in articles:
            url = art.get('url')
            if not url:
                continue
            if url not in cache:
                cache[url] = self._fetch_og_description(url)
            desc = cache[url]
            if desc and len(desc) >= 20:
                art['description'] = desc
        self.log('Found: ', len(articles), ' articles.\n')
        return articles

    def _fetch_og_description(self, url):
        try:
            soup = self.index_to_soup(url)
        except Exception as e:
            self.log.warn('LAT og:description fetch failed for', url, ':', e)
            return None
        meta = soup.find('meta', attrs={'property': 'og:description'})
        if meta and meta.get('content'):
            return meta['content'].strip() or None
        return None
