package org.jabref.logic.importer.fetcher;

import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import java.util.Optional;
import java.util.StringJoiner;
import java.util.stream.Collectors;

import javafx.collections.FXCollections;

import org.jabref.logic.bibtex.FieldPreferences;
import org.jabref.logic.importer.FetcherException;
import org.jabref.logic.importer.ImportCleanup;
import org.jabref.logic.importer.ImportFormatPreferences;
import org.jabref.logic.importer.PagedSearchBasedFetcher;
import org.jabref.logic.importer.SearchBasedFetcher;
import org.jabref.logic.util.URLUtil;
import org.jabref.model.database.BibDatabaseMode;
import org.jabref.model.entry.BibEntry;
import org.jabref.model.entry.field.InternalField;
import org.jabref.model.entry.field.StandardField;
import org.jabref.model.entry.field.UnknownField;
import org.jabref.model.entry.identifier.ArXivIdentifier;
import org.jabref.model.entry.types.StandardEntryType;
import org.jabref.testutils.category.FetcherTest;

import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.mockito.Answers;
import org.mockito.Mockito;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;

@FetcherTest
class ArXivFetcherTest implements SearchBasedFetcherCapabilityTest, PagedSearchFetcherTest {
    private static ImportFormatPreferences importFormatPreferences;

    private ArXivFetcher fetcher;
    private BibEntry entry;
    private BibEntry sliceTheoremPaper;

    private BibEntry mainOriginalPaper;
    private BibEntry mainResultPaper;

    @BeforeAll
    static void setUp() {
        importFormatPreferences = mock(ImportFormatPreferences.class, Answers.RETURNS_DEEP_STUBS);
        when(importFormatPreferences.bibEntryPreferences().getKeywordSeparator()).thenReturn(',');
        // Used during DOI fetch process
        when(importFormatPreferences.fieldPreferences().getNonWrappableFields()).thenReturn(
                FXCollections.observableArrayList(List.of(
                        StandardField.PDF,
                        StandardField.PS,
                        StandardField.URL,
                        StandardField.DOI,
                        StandardField.FILE,
                        StandardField.ISBN,
                        StandardField.ISSN)));
    }

    @BeforeEach
    void eachSetUp() {
        fetcher = new ArXivFetcher(importFormatPreferences);
        entry = new BibEntry();

        // A BibEntry with information only from ArXiv API
        mainOriginalPaper = new BibEntry(StandardEntryType.Article)
                // ArXiv-original fields
                .withField(StandardField.AUTHOR, "Joeran Beel and Andrew Collins and Akiko Aizawa")
                .withField(StandardField.TITLE, "The Architecture of Mr. DLib's Scientific Recommender-System API")
                .withField(StandardField.DATE, "2018-11-26")
                .withField(StandardField.ABSTRACT, "Recommender systems in academia are not widely available. This may be in part due to the difficulty and cost of developing and maintaining recommender systems. Many operators of academic products such as digital libraries and reference managers avoid this effort, although a recommender system could provide significant benefits to their users. In this paper, we introduce Mr. DLib's \"Recommendations as-a-Service\" (RaaS) API that allows operators of academic products to easily integrate a scientific recommender system into their products. Mr. DLib generates recommendations for research articles but in the future, recommendations may include call for papers, grants, etc. Operators of academic products can request recommendations from Mr. DLib and display these recommendations to their users. Mr. DLib can be integrated in just a few hours or days; creating an equivalent recommender system from scratch would require several months for an academic operator. Mr. DLib has been used by GESIS Sowiport and by the reference manager JabRef. Mr. DLib is open source and its goal is to facilitate the application of, and research on, scientific recommender systems. In this paper, we present the motivation for Mr. DLib, the architecture and details about the effectiveness. Mr. DLib has delivered 94m recommendations over a span of two years with an average click-through rate of 0.12%.")
                .withField(StandardField.EPRINT, "1811.10364")
                .withField(StandardField.FILE, ":https\\://arxiv.org/pdf/1811.10364v1:PDF")
                .withField(StandardField.EPRINTTYPE, "arXiv")
                .withField(StandardField.EPRINTCLASS, "cs.IR")
                .withField(StandardField.KEYWORDS, "cs.IR, cs.AI, cs.DL, cs.LG");

        mainResultPaper = new BibEntry(StandardEntryType.Article)
                // ArXiv-original fields
                //              .withField(StandardField.AUTHOR, "Joeran Beel and Andrew Collins and Akiko Aizawa")
                .withField(StandardField.TITLE, "The Architecture of Mr. DLib's Scientific Recommender-System API")
                .withField(StandardField.DATE, "2018-11-26")
                .withField(StandardField.ABSTRACT, "Recommender systems in academia are not widely available. This may be in part due to the difficulty and cost of developing and maintaining recommender systems. Many operators of academic products such as digital libraries and reference managers avoid this effort, although a recommender system could provide significant benefits to their users. In this paper, we introduce Mr. DLib's \"Recommendations as-a-Service\" (RaaS) API that allows operators of academic products to easily integrate a scientific recommender system into their products. Mr. DLib generates recommendations for research articles but in the future, recommendations may include call for papers, grants, etc. Operators of academic products can request recommendations from Mr. DLib and display these recommendations to their users. Mr. DLib can be integrated in just a few hours or days; creating an equivalent recommender system from scratch would require several months for an academic operator. Mr. DLib has been used by GESIS Sowiport and by the reference manager JabRef. Mr. DLib is open source and its goal is to facilitate the application of, and research on, scientific recommender systems. In this paper, we present the motivation for Mr. DLib, the architecture and details about the effectiveness. Mr. DLib has delivered 94m recommendations over a span of two years with an average click-through rate of 0.12%.")
                .withField(StandardField.EPRINT, "1811.10364")
                .withField(StandardField.FILE, ":https\\://arxiv.org/pdf/1811.10364v1:PDF")
                .withField(StandardField.EPRINTTYPE, "arXiv")
                .withField(StandardField.EPRINTCLASS, "cs.IR")
                //              .withField(StandardField.KEYWORDS, "cs.IR, cs.AI, cs.DL, cs.LG")
                // Unavailable info:
                // StandardField.JOURNALTITLE // INFO NOT APPLICABLE TO THIS ENTRY
                // ArXiv-issue DOI fields
                .withField(new UnknownField("copyright"), "arXiv.org perpetual, non-exclusive license")
                .withField(InternalField.KEY_FIELD, "https://doi.org/10.48550/arxiv.1811.10364")
                .withField(StandardField.YEAR, "2018")
                .withField(StandardField.KEYWORDS, "Information Retrieval (cs.IR), Artificial Intelligence (cs.AI), Digital Libraries (cs.DL), Machine Learning (cs.LG), FOS: Computer and information sciences")
                .withField(StandardField.AUTHOR, "Beel, Joeran and Collins, Andrew and Aizawa, Akiko")
                .withField(StandardField.PUBLISHER, "arXiv")
                .withField(StandardField.DOI, "10.48550/ARXIV.1811.10364");

        sliceTheoremPaper = new BibEntry(StandardEntryType.Article)
                .withField(StandardField.AUTHOR, "Diez, Tobias")
                .withField(StandardField.TITLE, "Slice theorem for Fréchet group actions and covariant symplectic field theory")
                .withField(StandardField.DATE, "2014-05-09")
                .withField(StandardField.YEAR, "2014")
                .withField(StandardField.PUBLISHER, "arXiv")
                .withField(StandardField.ABSTRACT, "A general slice theorem for the action of a Fréchet Lie group on a Fréchet manifolds is established. The Nash-Moser theorem provides the fundamental tool to generalize the result of Palais to this infinite-dimensional setting. The presented slice theorem is illustrated by its application to gauge theories: the action of the gauge transformation group admits smooth slices at every point and thus the gauge orbit space is stratified by Fréchet manifolds. Furthermore, a covariant and symplectic formulation of classical field theory is proposed and extensively discussed. At the root of this novel framework is the incorporation of field degrees of freedom F and spacetime M into the product manifold F * M. The induced bigrading of differential forms is used in order to carry over the usual symplectic theory to this new setting. The examples of the Klein-Gordon field and general Yang-Mills theory illustrate that the presented approach conveniently handles the occurring symmetries.")
                .withField(StandardField.DOI, "10.48550/ARXIV.1405.2249")
                .withField(StandardField.EPRINT, "1405.2249")
                .withField(StandardField.EPRINTCLASS, "math-ph")
                .withField(StandardField.EPRINTTYPE, "arXiv")
                .withField(StandardField.FILE, ":https\\://arxiv.org/pdf/1405.2249v1:PDF")
                .withField(StandardField.KEYWORDS, "Mathematical Physics (math-ph), Differential Geometry (math.DG), Symplectic Geometry (math.SG), FOS: Physical sciences, FOS: Mathematics, 58B99, 58Z05, 58B25, 22E65, 58D19, 53D20, 53D42")
                .withField(InternalField.KEY_FIELD, "Diez:2013fdp")
                .withField(new UnknownField("copyright"), "arXiv.org perpetual, non-exclusive license");
    }

    @Override
    public SearchBasedFetcher getFetcher() {
        return fetcher;
    }

    public List<String> getInputTestAuthors() {
        return Arrays.stream(mainOriginalPaper.getField(StandardField.AUTHOR).get()
                                              .split("and")).map(String::trim).collect(Collectors.toList());
    }

    @Override
    public List<String> getTestAuthors() {
        return Arrays.stream(mainResultPaper.getField(StandardField.AUTHOR).get()
                                            .split("and")).map(String::trim).collect(Collectors.toList());
    }

    @Override
    public String getTestJournal() {
        return "Journal of Geometry and Physics (2013)";
    }

    @Override
    public PagedSearchBasedFetcher getPagedFetcher() {
        return fetcher;
    }

    @Test
    @Override
    public void supportsAuthorSearch() throws FetcherException {
        StringJoiner queryBuilder = new StringJoiner("\" AND author=\"", "author=\"", "\"");
        getInputTestAuthors().forEach(queryBuilder::add);

        List<BibEntry> result = getFetcher().performSearch(queryBuilder.toString());

        FieldPreferences fieldPreferences = mock(FieldPreferences.class);
        when(fieldPreferences.getNonWrappableFields()).thenReturn(FXCollections.observableArrayList());
        ImportCleanup.targeting(BibDatabaseMode.BIBTEX, fieldPreferences).doPostCleanup(result);

        assertFalse(result.isEmpty());
        result.forEach(bibEntry -> {
            String author = bibEntry.getField(StandardField.AUTHOR).orElse("");

            // The co-authors differ, thus we check for the author present at all papers
            getTestAuthors().forEach(expectedAuthor -> assertTrue(author.contains(expectedAuthor.replace("\"", ""))));
        });
    }

    @Test
    void supportsAuthorSearchWithLastFirstName() throws FetcherException {
        StringJoiner queryBuilder = new StringJoiner("\" AND author:\"", "author:\"", "\"");
        getTestAuthors().forEach(queryBuilder::add);

        List<BibEntry> result = getFetcher().performSearch(queryBuilder.toString());
        FieldPreferences fieldPreferences = mock(FieldPreferences.class);
        when(fieldPreferences.getNonWrappableFields()).thenReturn(FXCollections.observableArrayList());
        ImportCleanup.targeting(BibDatabaseMode.BIBTEX, fieldPreferences).doPostCleanup(result);

        assertNotEquals(List.of(), result);
    }

    @Test
    void findFullTextForEmptyEntryResultsEmptyOptional() throws IOException {
        assertEquals(Optional.empty(), fetcher.findFullText(entry));
    }

    @Test
    void findFullTextRejectsNullParameter() {
        assertThrows(NullPointerException.class, () -> fetcher.findFullText(null));
    }

    @Test
    void findFullTextByDOI() throws IOException {
        entry.setField(StandardField.DOI, "10.1529/biophysj.104.047340");
        entry.setField(StandardField.TITLE, "Pause Point Spectra in DNA Constant-Force Unzipping");

        assertEquals(Optional.of(URLUtil.create("https://arxiv.org/pdf/cond-mat/0406246v1")), fetcher.findFullText(entry));
    }

    @Test
    void findFullTextByEprint() throws IOException {
        entry.setField(StandardField.EPRINT, "1603.06570");
        assertEquals(Optional.of(URLUtil.create("https://arxiv.org/pdf/1603.06570v1")), fetcher.findFullText(entry));
    }

    @Test
    void findFullTextByEprintWithPrefix() throws IOException {
        entry.setField(StandardField.EPRINT, "arXiv:1603.06570");
        assertEquals(Optional.of(URLUtil.create("https://arxiv.org/pdf/1603.06570v1")), fetcher.findFullText(entry));
    }

    @Test
    void findFullTextByEprintWithUnknownDOI() throws IOException {
        entry.setField(StandardField.DOI, "10.1529/unknown");
        entry.setField(StandardField.EPRINT, "1603.06570");

        assertEquals(Optional.of(URLUtil.create("https://arxiv.org/pdf/1603.06570v1")), fetcher.findFullText(entry));
    }

    @Test
    void findFullTextByTitle() throws IOException {
        entry.setField(StandardField.TITLE, "Pause Point Spectra in DNA Constant-Force Unzipping");

        assertEquals(Optional.of(URLUtil.create("https://arxiv.org/pdf/cond-mat/0406246v1")), fetcher.findFullText(entry));
    }

    @Test
    void findFullTextByTitleWithCurlyBracket() throws IOException {
        entry.setField(StandardField.TITLE, "Machine versus {Human} {Attention} in {Deep} {Reinforcement} {Learning} {Tasks}");

        assertEquals(Optional.of(URLUtil.create("https://arxiv.org/pdf/2010.15942v3")), fetcher.findFullText(entry));
    }

    @Test
    void findFullTextByTitleWithColonAndJournalWithoutEprint() throws IOException {
        entry.setField(StandardField.TITLE, "Bayes-TrEx: a Bayesian Sampling Approach to Model Transparency by Example");
        entry.setField(StandardField.JOURNAL, "arXiv:2002.10248v4 [cs]");

        assertEquals(Optional.of(URLUtil.create("https://arxiv.org/pdf/2002.10248v4")), fetcher.findFullText(entry));
    }

    @Test
    void findFullTextByTitleWithColonAndUrlWithoutEprint() throws IOException {
        entry.setField(StandardField.TITLE, "Bayes-TrEx: a Bayesian Sampling Approach to Model Transparency by Example");
        entry.setField(StandardField.URL, "https://arxiv.org/abs/2002.10248v4");

        assertEquals(Optional.of(URLUtil.create("https://arxiv.org/pdf/2002.10248v4")), fetcher.findFullText(entry));
    }

    @Test
    void findFullTextByTitleAndPartOfAuthor() throws IOException {
        entry.setField(StandardField.TITLE, "Pause Point Spectra in DNA Constant-Force Unzipping");
        entry.setField(StandardField.AUTHOR, "Weeks and Lucks");

        assertEquals(Optional.of(URLUtil.create("https://arxiv.org/pdf/cond-mat/0406246v1")), fetcher.findFullText(entry));
    }

    @Test
    void findFullTextByTitleWithCurlyBracketAndPartOfAuthor() throws IOException {
        entry.setField(StandardField.TITLE, "Machine versus {Human} {Attention} in {Deep} {Reinforcement} {Learning} {Tasks}");
        entry.setField(StandardField.AUTHOR, "Zhang, Ruohan and Guo");

        assertEquals(Optional.of(URLUtil.create("https://arxiv.org/pdf/2010.15942v3")), fetcher.findFullText(entry));
    }

    @Test
    void notFindFullTextByUnknownDOI() throws IOException {
        entry.setField(StandardField.DOI, "10.1529/unknown");
        assertEquals(Optional.empty(), fetcher.findFullText(entry));
    }

    @Test
    void notFindFullTextByUnknownId() throws IOException {
        entry.setField(StandardField.EPRINT, "1234.12345");
        assertEquals(Optional.empty(), fetcher.findFullText(entry));
    }

    @Test
    void findFullTextByDOINotAvailableInCatalog() throws IOException {
        entry.setField(StandardField.DOI, "10.1016/0370-2693(77)90015-6");
        entry.setField(StandardField.TITLE, "Superspace formulation of supergravity");

        assertEquals(Optional.empty(), fetcher.findFullText(entry));
    }

    @Test
    void findFullTextEntityWithoutDoi() throws IOException {
        assertEquals(Optional.empty(), fetcher.findFullText(entry));
    }

    @Test
    void findFullTextTrustLevel() {
        assertEquals(TrustLevel.PREPRINT, fetcher.getTrustLevel());
    }

    @Test
    void searchEntryByPartOfTitle() throws FetcherException {
        assertEquals(List.of(mainResultPaper),
                fetcher.performSearch("title=\"the architecture of mr. dLib's\""));
    }

    @Test
    void searchEntryByPartOfTitleWithAcuteAccent() throws FetcherException {
        assertEquals(List.of(sliceTheoremPaper),
                fetcher.performSearch("title=\"slice theorem for Fréchet\""));
    }

    @Test
    void searchEntryByOldId() throws FetcherException {
        BibEntry expected = new BibEntry(StandardEntryType.Article)
                .withField(StandardField.AUTHOR, "{H1 Collaboration}")
                .withField(StandardField.TITLE, "Multi-Electron Production at High Transverse Momenta in ep Collisions at HERA")
                .withField(StandardField.NUMBER, "1")
                .withField(StandardField.VOLUME, "31")
                .withField(StandardField.PAGES, "17--29")
                .withField(StandardField.DATE, "2003-07-07")
                .withField(StandardField.YEAR, "2003")
                .withField(StandardField.MONTH, "#oct#")
                .withField(StandardField.ISSN, "1434-6052")
                .withField(StandardField.ABSTRACT, "Multi-electron production is studied at high electron transverse momentum in positron- and electron-proton collisions using the H1 detector at HERA. The data correspond to an integrated luminosity of 115 pb-1. Di-electron and tri-electron event yields are measured. Cross sections are derived in a restricted phase space region dominated by photon-photon collisions. In general good agreement is found with the Standard Model predictions. However, for electron pair invariant masses above 100 GeV, three di-electron events and three tri-electron events are observed, compared to Standard Model expectations of 0.30 \\pm 0.04 and 0.23 \\pm 0.04, respectively.")
                .withField(StandardField.PUBLISHER, "Springer Science and Business Media LLC")
                .withField(StandardField.EPRINT, "hep-ex/0307015")
                .withField(StandardField.FILE, ":https\\://arxiv.org/pdf/hep-ex/0307015v1:PDF")
                .withField(StandardField.EPRINTTYPE, "arXiv")
                .withField(StandardField.EPRINTCLASS, "hep-ex")
                .withField(StandardField.KEYWORDS, "High Energy Physics - Experiment (hep-ex), FOS: Physical sciences")
                .withField(StandardField.DOI, "10.1140/epjc/s2003-01326-x")
                .withField(StandardField.JOURNAL, "Eur.Phys.J.C31:17-29,2003")
                .withField(InternalField.KEY_FIELD, "2003")
                .withField(new UnknownField("copyright"), "Assumed arXiv.org perpetual, non-exclusive license to distribute this article for submissions made before January 2004");

        assertEquals(Optional.of(expected), fetcher.performSearchById("hep-ex/0307015"));
    }

    @Test
    void searchEntryByIdWith4DigitsAndVersion() throws FetcherException {
        assertEquals(Optional.of(sliceTheoremPaper), fetcher.performSearchById("1405.2249v1"));
    }

    @Test
    void searchEntryByIdWith4Digits() throws FetcherException {
        assertEquals(Optional.of(sliceTheoremPaper), fetcher.performSearchById("1405.2249"));
    }

    @Test
    void citationKeyIsAdoptedFromInspireWhenPaperIsIndexedThere() throws FetcherException {
        // 1405.2249 has no manually-assigned (journal) DOI, so ArXivFetcher's own DOI-based
        // enrichment never produces a citation key for it; but it is indexed on INSPIRE, which
        // should supply its curated texkey instead of leaving the entry keyless.
        Optional<BibEntry> entry = fetcher.performSearchById("1405.2249");
        assertEquals(Optional.of("Diez:2013fdp"), entry.flatMap(BibEntry::getCitationKey));
    }

    @Test
    void urlShapedCitationKeyIsLeftUnchangedWhenPaperIsNotIndexedOnInspire() throws FetcherException {
        // 1811.10364 (mainResultPaper, a cs.IR paper) has no manually-assigned DOI, so it ends up
        // with a URL-shaped key from the automatically-assigned arXiv DOI's metadata. It is not
        // indexed on INSPIRE (a physics-only database), so that key is left as-is.
        Optional<BibEntry> entry = fetcher.performSearchById("1811.10364");
        assertEquals(Optional.of("https://doi.org/10.48550/arxiv.1811.10364"), entry.flatMap(BibEntry::getCitationKey));
    }

    @Test
    void searchEntryByIdWith4DigitsAndPrefix() throws FetcherException {
        assertEquals(Optional.of(sliceTheoremPaper), fetcher.performSearchById("arXiv:1405.2249"));
    }

    @Test
    void searchEntryByIdWith4DigitsAndPrefixAndNotTrimmed() throws FetcherException {
        assertEquals(Optional.of(sliceTheoremPaper), fetcher.performSearchById("arXiv : 1405. 2249"));
    }

    @Test
    void searchEntryByIdWith5Digits() throws FetcherException {
        assertEquals(Optional.of(
                        "An Optimal Convergence Theorem for Mean Curvature Flow of Arbitrary Codimension in Hyperbolic Spaces"),
                fetcher.performSearchById("1503.06747").flatMap(entry -> entry.getField(StandardField.TITLE)));
    }

    @Test
    void searchWithMalformedIdReturnsEmpty() throws FetcherException {
        assertEquals(Optional.empty(), fetcher.performSearchById("123412345"));
    }

    @Test
    void searchIdentifierForSlicePaper() throws FetcherException {
        sliceTheoremPaper.clearField(StandardField.EPRINT);

        assertEquals(ArXivIdentifier.parse("1405.2249"), fetcher.findIdentifier(sliceTheoremPaper));
    }

    @Test
    void searchIdentifierForSlicePaperByDoiUrl() throws FetcherException {
        sliceTheoremPaper.clearField(StandardField.EPRINT);
        sliceTheoremPaper.setField(StandardField.DOI, "https://doi.org/10.48550/arXiv.1405.2249");

        assertEquals(ArXivIdentifier.parse("1405.2249"), fetcher.findIdentifier(sliceTheoremPaper));
    }

    @Test
    void searchEmptyId() throws FetcherException {
        assertEquals(Optional.empty(), fetcher.performSearchById(""));
    }

    @Test
    void searchWithHttpUrl() throws FetcherException {
        assertEquals(Optional.of(sliceTheoremPaper), fetcher.performSearchById("https://arxiv.org/abs/1405.2249"));
    }

    @Test
    void searchWithHttpsUrl() throws FetcherException {
        assertEquals(Optional.of(sliceTheoremPaper), fetcher.performSearchById("https://arxiv.org/abs/1405.2249"));
    }

    @Test
    void searchWithHttpsUrlNotTrimmed() throws FetcherException {
        assertEquals(Optional.of(sliceTheoremPaper), fetcher.performSearchById("https : // arxiv . org / abs / 1405 . 2249 "));
    }

    @Disabled("Is not supported by the current API")
    @Test
    @Override
    public void supportsYearSearch() {
    }

    @Disabled("Is not supported by the current API")
    @Test
    @Override
    public void supportsYearRangeSearch() {
    }

    /// A phrase is a sequence of terms wrapped in quotes.
    /// Only documents that contain exactly this sequence are returned.
    @Test
    void supportsPhraseSearch() throws FetcherException {
        List<BibEntry> resultWithPhraseSearch = fetcher.performSearch("title:\"Taxonomy of Distributed\"");
        List<BibEntry> broaderSearchResult = fetcher.performSearch("taxonomy distributed");

        assertFalse(resultWithPhraseSearch.isEmpty());
        List<Optional<String>> broaderSearchIdentifiers = broaderSearchResult.stream()
                                                                             .map(entry -> entry.getField(StandardField.EPRINT))
                                                                             .toList();
        assertTrue(resultWithPhraseSearch.stream()
                                         .map(entry -> entry.getField(StandardField.EPRINT))
                                         .allMatch(broaderSearchIdentifiers::contains));
    }

    /// A phrase is a sequence of terms wrapped in quotes.
    /// Only documents that contain exactly this sequence are returned.
    @Test
    void supportsPhraseSearchAndMatchesExact() throws FetcherException {
        BibEntry expected = new BibEntry(StandardEntryType.Article)
                .withField(StandardField.AUTHOR, "Rafrastara, Fauzi Adi and Deyu, Qi")
                .withField(StandardField.TITLE, "A Survey and Taxonomy of Distributed Data Mining Research Studies: A Systematic Literature Review")
                .withField(StandardField.DATE, "2020-09-14")
                .withField(StandardField.YEAR, "2020")
                .withField(StandardField.PUBLISHER, "arXiv")
                .withField(StandardField.ABSTRACT, "Context: Data Mining (DM) method has been evolving year by year and as of today there is also the enhancement of DM technique that can be run several times faster than the traditional one, called Distributed Data Mining (DDM). It is not a new field in data processing actually, but in the recent years many researchers have been paying more attention on this area. Problems: The number of publication regarding DDM in high reputation journals and conferences has increased significantly. It makes difficult for researchers to gain a comprehensive view of DDM that require further research. Solution: We conducted a systematic literature review to map the previous research in DDM field. Our objective is to provide the motivation for new research by identifying the gap in DDM field as well as the hot area itself. Result: Our analysis came up with some conclusions by answering 7 research questions proposed in this literature review. In addition, the taxonomy of DDM research area is presented in this paper. Finally, this systematic literature review provides the statistic of development of DDM since 2000 to 2015, in which this will help the future researchers to have a comprehensive overview of current situation of DDM.")
                .withField(StandardField.EPRINT, "2009.10618")
                .withField(StandardField.DOI, "10.48550/ARXIV.2009.10618")
                .withField(StandardField.FILE, ":https\\://arxiv.org/pdf/2009.10618v1:PDF")
                .withField(StandardField.EPRINTTYPE, "arXiv")
                .withField(StandardField.EPRINTCLASS, "cs.DC")
                .withField(StandardField.KEYWORDS, "Distributed / Parallel / Cluster Computing (cs.DC), Machine Learning (cs.LG), FOS: Computer and information sciences")
                .withField(InternalField.KEY_FIELD, "https://doi.org/10.48550/arxiv.2009.10618")
                .withField(new UnknownField("copyright"), "arXiv.org perpetual, non-exclusive license");

        List<BibEntry> resultWithPhraseSearch = fetcher.performSearch("title==\"A Survey and Taxonomy of Distributed Data Mining Research Studies: A Systematic Literature Review\"");

        // The first result should be the expected paper
        assertEquals(expected, resultWithPhraseSearch.getFirst());
    }

    @Test
    void supportsBooleanANDSearch() throws FetcherException {
        // Example of a robust result, with information from both ArXiv-assigned and user-assigned DOIs
        BibEntry expected = new BibEntry(StandardEntryType.Article)
                .withField(StandardField.AUTHOR, "Büscher, Tobias and Diez, Angel L. and Gompper, Gerhard and Elgeti, Jens")
                .withField(StandardField.TITLE, "Instability and fingering of interfaces in growing tissue")
                .withField(StandardField.DATE, "2020-03-10")
                .withField(StandardField.YEAR, "2020")
                .withField(StandardField.MONTH, "#aug#")
                .withField(StandardField.NUMBER, "8")
                .withField(StandardField.VOLUME, "22")
                .withField(StandardField.ISSN, "1367-2630")
                .withField(StandardField.PAGES, "083005")
                .withField(StandardField.PUBLISHER, "IOP Publishing")
                .withField(StandardField.JOURNAL, "New J. Phys., 22, 083005 (2020)")
                .withField(StandardField.ABSTRACT, "Interfaces in tissues are ubiquitous, both between tissue and environment as well as between populations of different cell types. The propagation of an interface can be driven mechanically. % e.g. by a difference in the respective homeostatic stress of the different cell types. Computer simulations of growing tissues are employed to study the stability of the interface between two tissues on a substrate. From a mechanical perspective, the dynamics and stability of this system is controlled mainly by four parameters of the respective tissues: (i) the homeostatic stress (ii) cell motility (iii) tissue viscosity and (iv) substrate friction. For propagation driven by a difference in homeostatic stress, the interface is stable for tissue-specific substrate friction even for very large differences of homeostatic stress; however, it becomes unstable above a critical stress difference when the tissue with the larger homeostatic stress has a higher viscosity. A small difference in directed bulk motility between the two tissues suffices to result in propagation with a stable interface, even for otherwise identical tissues. Larger differences in motility force, however, result in a finite-wavelength instability of the interface. Interestingly, the instability is apparently bound by nonlinear effects and the amplitude of the interface undulations only grows to a finite value in time.")
                .withField(StandardField.DOI, "10.1088/1367-2630/ab9e88")
                .withField(StandardField.EPRINT, "2003.04601")
                .withField(StandardField.FILE, ":https\\://arxiv.org/pdf/2003.04601v1:PDF")
                .withField(StandardField.EPRINTTYPE, "arXiv")
                .withField(StandardField.EPRINTCLASS, "q-bio.TO")
                .withField(StandardField.KEYWORDS, "Tissues and Organs (q-bio.TO), FOS: Biological sciences")
                .withField(InternalField.KEY_FIELD, "B_scher_2020")
                .withField(new UnknownField("copyright"), "arXiv.org perpetual, non-exclusive license");

        List<BibEntry> result = fetcher.performSearch("author=\"Tobias Büscher\" AND title=\"Instability and fingering of interfaces\"");

        // There is only one paper authored by Tobias Büscher with that phrase in the title
        assertEquals(List.of(expected), result);
    }

    @Test
    void retrievePureArxivEntryWhenAllDOIFetchingFails() throws FetcherException {
        BibEntry expected = new BibEntry(StandardEntryType.Article)
                .withField(StandardField.AUTHOR, "Hai Zheng and Po-Yi Ho and Meiling Jiang and Bin Tang and Weirong Liu and Dengjin Li and Xuefeng Yu and Nancy E. Kleckner and Ariel Amir and Chenli Liu")
                .withField(StandardField.TITLE, "Interrogating the Escherichia coli cell cycle by cell dimension perturbations")
                .withField(StandardField.DATE, "2017-01-03")
                .withField(StandardField.JOURNAL, "PNAS December 27, 2016 vol. 113 no. 52 15000-15005")
                .withField(StandardField.ABSTRACT, "Bacteria tightly regulate and coordinate the various events in their cell cycles to duplicate themselves accurately and to control their cell sizes. Growth of Escherichia coli, in particular, follows a relation known as Schaechter 's growth law. This law says that the average cell volume scales exponentially with growth rate, with a scaling exponent equal to the time from initiation of a round of DNA replication to the cell division at which the corresponding sister chromosomes segregate. Here, we sought to test the robustness of the growth law to systematic perturbations in cell dimensions achieved by varying the expression levels of mreB and ftsZ. We found that decreasing the mreB level resulted in increased cell width, with little change in cell length, whereas decreasing the ftsZ level resulted in increased cell length. Furthermore, the time from replication termination to cell division increased with the perturbed dimension in both cases. Moreover, the growth law remained valid over a range of growth conditions and dimension perturbations. The growth law can be quantitatively interpreted as a consequence of a tight coupling of cell division to replication initiation. Thus, its robustness to perturbations in cell dimensions strongly supports models in which the timing of replication initiation governs that of cell division, and cell volume is the key phenomenological variable governing the timing of replication initiation. These conclusions are discussed in the context of our recently proposed adder-per-origin model, in which cells add a constant volume per origin between initiations and divide a constant time after initiation.")
                .withField(StandardField.DOI, "10.1073/pnas.1617932114")
                .withField(StandardField.EPRINT, "1701.00587")
                .withField(StandardField.FILE, ":https\\://arxiv.org/pdf/1701.00587v1:PDF")
                .withField(StandardField.EPRINTTYPE, "arXiv")
                .withField(StandardField.EPRINTCLASS, "q-bio.CB")
                .withField(StandardField.KEYWORDS, "q-bio.CB");

        DoiFetcher modifiedDoiFetcher = Mockito.spy(new DoiFetcher(importFormatPreferences));
        when(modifiedDoiFetcher.performSearchById("10.1073/pnas.1617932114")).thenThrow(new FetcherException("Could not fetch user-assigned DOI"));
        when(modifiedDoiFetcher.performSearchById("10.48550/arXiv.1701.00587")).thenThrow(new FetcherException("Could not fetch ArXiv-assigned DOI"));

        ArXivFetcher modifiedArXivFetcher = Mockito.spy(new ArXivFetcher(importFormatPreferences, modifiedDoiFetcher));
        assertEquals(Optional.of(expected), modifiedArXivFetcher.performSearchById("1701.00587"));
    }

    @Test
    void canReplicateArXivOnlySearchByPassingNullParameter() throws FetcherException {
        BibEntry expected = new BibEntry(StandardEntryType.Article)
                .withField(StandardField.AUTHOR, "Hai Zheng and Po-Yi Ho and Meiling Jiang and Bin Tang and Weirong Liu and Dengjin Li and Xuefeng Yu and Nancy E. Kleckner and Ariel Amir and Chenli Liu")
                .withField(StandardField.TITLE, "Interrogating the Escherichia coli cell cycle by cell dimension perturbations")
                .withField(StandardField.DATE, "2017-01-03")
                .withField(StandardField.JOURNAL, "PNAS December 27, 2016 vol. 113 no. 52 15000-15005")
                .withField(StandardField.ABSTRACT, "Bacteria tightly regulate and coordinate the various events in their cell cycles to duplicate themselves accurately and to control their cell sizes. Growth of Escherichia coli, in particular, follows a relation known as Schaechter 's growth law. This law says that the average cell volume scales exponentially with growth rate, with a scaling exponent equal to the time from initiation of a round of DNA replication to the cell division at which the corresponding sister chromosomes segregate. Here, we sought to test the robustness of the growth law to systematic perturbations in cell dimensions achieved by varying the expression levels of mreB and ftsZ. We found that decreasing the mreB level resulted in increased cell width, with little change in cell length, whereas decreasing the ftsZ level resulted in increased cell length. Furthermore, the time from replication termination to cell division increased with the perturbed dimension in both cases. Moreover, the growth law remained valid over a range of growth conditions and dimension perturbations. The growth law can be quantitatively interpreted as a consequence of a tight coupling of cell division to replication initiation. Thus, its robustness to perturbations in cell dimensions strongly supports models in which the timing of replication initiation governs that of cell division, and cell volume is the key phenomenological variable governing the timing of replication initiation. These conclusions are discussed in the context of our recently proposed adder-per-origin model, in which cells add a constant volume per origin between initiations and divide a constant time after initiation.")
                .withField(StandardField.DOI, "10.1073/pnas.1617932114")
                .withField(StandardField.EPRINT, "1701.00587")
                .withField(StandardField.FILE, ":https\\://arxiv.org/pdf/1701.00587v1:PDF")
                .withField(StandardField.EPRINTTYPE, "arXiv")
                .withField(StandardField.EPRINTCLASS, "q-bio.CB")
                .withField(StandardField.KEYWORDS, "q-bio.CB");

        ArXivFetcher modifiedArXivFetcher = new ArXivFetcher(importFormatPreferences, null);
        assertEquals(Optional.of(expected), modifiedArXivFetcher.performSearchById("1701.00587"));
    }

    @Test
    void retrievePartialResultWhenCannotGetInformationFromUserAssignedDOI() throws FetcherException {
        BibEntry expected = new BibEntry(StandardEntryType.Article)
                .withField(StandardField.AUTHOR, "Zheng, Hai and Ho, Po-Yi and Jiang, Meiling and Tang, Bin and Liu, Weirong and Li, Dengjin and Yu, Xuefeng and Kleckner, Nancy E. and Amir, Ariel and Liu, Chenli")
                .withField(StandardField.TITLE, "Interrogating the Escherichia coli cell cycle by cell dimension perturbations")
                .withField(StandardField.DATE, "2017-01-03")
                .withField(StandardField.JOURNAL, "PNAS December 27, 2016 vol. 113 no. 52 15000-15005")
                .withField(StandardField.ABSTRACT, "Bacteria tightly regulate and coordinate the various events in their cell cycles to duplicate themselves accurately and to control their cell sizes. Growth of Escherichia coli, in particular, follows a relation known as Schaechter 's growth law. This law says that the average cell volume scales exponentially with growth rate, with a scaling exponent equal to the time from initiation of a round of DNA replication to the cell division at which the corresponding sister chromosomes segregate. Here, we sought to test the robustness of the growth law to systematic perturbations in cell dimensions achieved by varying the expression levels of mreB and ftsZ. We found that decreasing the mreB level resulted in increased cell width, with little change in cell length, whereas decreasing the ftsZ level resulted in increased cell length. Furthermore, the time from replication termination to cell division increased with the perturbed dimension in both cases. Moreover, the growth law remained valid over a range of growth conditions and dimension perturbations. The growth law can be quantitatively interpreted as a consequence of a tight coupling of cell division to replication initiation. Thus, its robustness to perturbations in cell dimensions strongly supports models in which the timing of replication initiation governs that of cell division, and cell volume is the key phenomenological variable governing the timing of replication initiation. These conclusions are discussed in the context of our recently proposed adder-per-origin model, in which cells add a constant volume per origin between initiations and divide a constant time after initiation.")
                .withField(StandardField.DOI, "10.1073/pnas.1617932114")
                .withField(StandardField.EPRINT, "1701.00587")
                .withField(StandardField.FILE, ":https\\://arxiv.org/pdf/1701.00587v1:PDF")
                .withField(StandardField.EPRINTTYPE, "arXiv")
                .withField(StandardField.EPRINTCLASS, "q-bio.CB")
                .withField(StandardField.KEYWORDS, "Cell Behavior (q-bio.CB), FOS: Biological sciences")
                .withField(new UnknownField("copyright"), "arXiv.org perpetual, non-exclusive license")
                .withField(InternalField.KEY_FIELD, "https://doi.org/10.48550/arxiv.1701.00587")
                .withField(StandardField.YEAR, "2017")
                .withField(StandardField.PUBLISHER, "arXiv");

        DoiFetcher modifiedDoiFetcher = Mockito.spy(new DoiFetcher(importFormatPreferences));
        when(modifiedDoiFetcher.performSearchById("10.1073/pnas.1617932114")).thenThrow(new FetcherException("Could not fetch user-assigned DOI"));

        ArXivFetcher modifiedArXivFetcher = Mockito.spy(new ArXivFetcher(importFormatPreferences, modifiedDoiFetcher));
        assertEquals(Optional.of(expected), modifiedArXivFetcher.performSearchById("1701.00587"));
    }

    @Test
    void retrievePartialResultWhenCannotGetInformationFromArXivAssignedDOI() throws FetcherException {
        BibEntry expected = new BibEntry(StandardEntryType.Article)
                .withField(StandardField.AUTHOR, "Hai Zheng and Po-Yi Ho and Meiling Jiang and Bin Tang and Weirong Liu and Dengjin Li and Xuefeng Yu and Nancy E. Kleckner and Ariel Amir and Chenli Liu")
                .withField(StandardField.TITLE, "Interrogating the Escherichia coli cell cycle by cell dimension perturbations")
                .withField(StandardField.DATE, "2017-01-03")
                .withField(StandardField.JOURNAL, "PNAS December 27, 2016 vol. 113 no. 52 15000-15005")
                .withField(StandardField.ABSTRACT, "Bacteria tightly regulate and coordinate the various events in their cell cycles to duplicate themselves accurately and to control their cell sizes. Growth of Escherichia coli, in particular, follows a relation known as Schaechter 's growth law. This law says that the average cell volume scales exponentially with growth rate, with a scaling exponent equal to the time from initiation of a round of DNA replication to the cell division at which the corresponding sister chromosomes segregate. Here, we sought to test the robustness of the growth law to systematic perturbations in cell dimensions achieved by varying the expression levels of mreB and ftsZ. We found that decreasing the mreB level resulted in increased cell width, with little change in cell length, whereas decreasing the ftsZ level resulted in increased cell length. Furthermore, the time from replication termination to cell division increased with the perturbed dimension in both cases. Moreover, the growth law remained valid over a range of growth conditions and dimension perturbations. The growth law can be quantitatively interpreted as a consequence of a tight coupling of cell division to replication initiation. Thus, its robustness to perturbations in cell dimensions strongly supports models in which the timing of replication initiation governs that of cell division, and cell volume is the key phenomenological variable governing the timing of replication initiation. These conclusions are discussed in the context of our recently proposed adder-per-origin model, in which cells add a constant volume per origin between initiations and divide a constant time after initiation.")
                .withField(StandardField.DOI, "10.1073/pnas.1617932114")
                .withField(StandardField.ISSN, "1091-6490")
                .withField(StandardField.EPRINT, "1701.00587")
                .withField(StandardField.FILE, ":https\\://arxiv.org/pdf/1701.00587v1:PDF")
                .withField(StandardField.EPRINTTYPE, "arXiv")
                .withField(StandardField.EPRINTCLASS, "q-bio.CB")
                .withField(StandardField.KEYWORDS, "q-bio.CB")
                .withField(StandardField.MONTH, "#dec#")
                .withField(StandardField.YEAR, "2016")
                .withField(StandardField.VOLUME, "113")
                .withField(InternalField.KEY_FIELD, "Zheng_2016")
                .withField(StandardField.PUBLISHER, "National Academy of Sciences")
                .withField(StandardField.PAGES, "15000--15005")
                .withField(StandardField.NUMBER, "52");

        DoiFetcher modifiedDoiFetcher = Mockito.spy(new DoiFetcher(importFormatPreferences));
        when(modifiedDoiFetcher.performSearchById("10.48550/arXiv.1701.00587")).thenThrow(new FetcherException("Could not fetch ArXiv-assigned DOI"));

        ArXivFetcher modifiedArXivFetcher = Mockito.spy(new ArXivFetcher(importFormatPreferences, modifiedDoiFetcher));
        assertEquals(Optional.of(expected), modifiedArXivFetcher.performSearchById("1701.00587"));
    }

    @Test
    void performRawSearchQueryPagedWithBlankQueryReturnsEmptyPage() throws FetcherException {
        assertTrue(fetcher.performRawSearchQueryPaged("", 0).getContent().isEmpty());
    }

    @Test
    void performRawSearchQueryPagedReturnsResults() throws FetcherException {
        assertFalse(fetcher.performRawSearchQueryPaged("machine learning", 0).getContent().isEmpty());
    }

    @Test
    void abstractIsCleanedUp() throws FetcherException {
        Optional<BibEntry> entry = fetcher.performSearchById("2407.02238");
        String escaped = "{One of the primary areas of interest in High Performance Computing is the improvement of performance of parallel workloads. Nowadays, compilable source code-based optimization tasks that employ deep learning often exploit LLVM Intermediate Representations (IRs) for extracting features from source code. Most such works target specific tasks, or are designed with a pre-defined set of heuristics. So far, pre-trained models are rare in this domain, but the possibilities have been widely discussed. Especially approaches mimicking large-language models (LLMs) have been proposed. But these have prohibitively large training costs. In this paper, we propose MIREncoder, a M}ulti-modal IR-based Auto-Encoder that can be pre-trained to generate a learned embedding space to be used for downstream tasks by machine learning-based approaches. A multi-modal approach enables us to better extract features from compilable programs. It allows us to better model code syntax, semantics and structure. For code-based performance optimizations, these features are very important while making optimization decisions. A pre-trained model/embedding implicitly enables the usage of transfer learning, and helps move away from task-specific trained models. Additionally, a pre-trained model used for downstream performance optimization should itself have reduced overhead, and be easily usable. These considerations have led us to propose a modeling approach that i) understands code semantics and structure, ii) enables use of transfer learning, and iii) is small and simple enough to be easily re-purposed or reused even with low resource availability. Our evaluations will show that our proposed approach can outperform the state of the art while reducing overhead.";
        assertEquals(Optional.of(escaped), entry.get().getField(StandardField.ABSTRACT));
    }
}
