#ifdef SORBET_REALMAIN_MIN
// minimal build to speedup compilation. Remove extra features
#else
#define FULL_BUILD_ONLY(X) X;
#include "common/statsd/statsd.h"
#include "common/web_tracer_framework/tracing.h"
#include "core/json/json.h"
#include "main/autogen/autogen.h"
#include "main/autogen/crc_builder.h"
#include "main/autogen/data/version.h"
#include "main/autogen/subclasses.h"
#include "main/lsp/LSPInput.h"
#include "main/lsp/LSPLoop.h"
#include "main/lsp/LSPOutput.h"
#include "main/minimize/minimize.h"
#include "packager/packager.h"
#endif

#include "absl/strings/match.h"
#include "absl/strings/str_cat.h"
#include "absl/strings/str_split.h"
#include "common/FileOps.h"
#include "common/concurrency/Parallel.h"
#include "common/timers/Timer.h"
#include "core/Error.h"
#include "core/ErrorQueue.h"
#include "core/Files.h"
#include "core/Unfreeze.h"
#include "core/errors/errors.h"
#include "core/serialize/serialize.h"
#include "hashing/hashing.h"
#include "main/cache/cache.h"
#include "main/pipeline/pipeline.h"
#include "main/realmain.h"
#include "packager/GenPackages.h"
#include "payload/payload.h"
#include "resolver/resolver.h"
#include "sorbet_version/sorbet_version.h"
#include "spdlog/sinks/rotating_file_sink.h"
#include "spdlog/sinks/stdout_color_sinks.h"

#include <csignal>
#include <iterator>
#include <poll.h>

using namespace std;

namespace sorbet::realmain {
shared_ptr<spdlog::logger> logger;
int returnCode;

shared_ptr<spdlog::sinks::ansicolor_stderr_sink_mt> make_stderrColorSink() {
    auto color_sink = make_shared<spdlog::sinks::ansicolor_stderr_sink_mt>();
    color_sink->set_color(spdlog::level::info, color_sink->white);
    color_sink->set_color(spdlog::level::debug, color_sink->magenta);
    color_sink->set_level(spdlog::level::info);
    return color_sink;
}

shared_ptr<spdlog::sinks::ansicolor_stderr_sink_mt> stderrColorSink = make_stderrColorSink();

/*
 * Workaround https://bugzilla.mindrot.org/show_bug.cgi?id=2863 ; We are
 * commonly run under ssh with a controlmaster, and we write exclusively to
 * STDERR in normal usage. If the client goes away, we can hang forever writing
 * to a full pipe buffer on stderr.
 *
 * Workaround by monitoring for STDOUT to go away and self-HUPing.
 */
void startHUPMonitor() {
    thread monitor([]() {
        struct pollfd pfd;
        setCurrentThreadName("HUPMonitor");
        pfd.fd = 1; // STDOUT
        pfd.events = 0;
        pfd.revents = 0;
        while (true) {
            int rv = poll(&pfd, 1, -1);
            if (rv <= 0) {
                continue;
            }
            if ((pfd.revents & (POLLHUP | POLLERR)) != 0) {
                // STDOUT has gone away; Exit via SIGHUP.
                kill(getpid(), SIGHUP);
            }
        }
    });
    monitor.detach();
}

core::StrictLevel levelMinusOne(core::StrictLevel level) {
    switch (level) {
        case core::StrictLevel::Ignore:
            return core::StrictLevel::None;
        case core::StrictLevel::False:
            return core::StrictLevel::Ignore;
        case core::StrictLevel::True:
            return core::StrictLevel::False;
        case core::StrictLevel::Strict:
            return core::StrictLevel::True;
        case core::StrictLevel::Strong:
            return core::StrictLevel::Strict;
        case core::StrictLevel::Max:
            return core::StrictLevel::Strong;
        default:
            Exception::raise("Should never attempt to subtract one from strictLevel={}", static_cast<uint8_t>(level));
    }
}

// Filter levels to a sensible recommendation.
core::StrictLevel levelToRecommendation(core::StrictLevel level) {
    switch (level) {
        case core::StrictLevel::Ignore:
            // We don't suggest `# typed: ignore` because it is too common for some problems in a
            // generated RBI file to cause a problem that actually does need to be fixed, and not by
            // ignoring the RBI file. Ignoring the file has bad consequences, like introducing more
            // errors in other files, causing those files to be ignored, etc.
            return core::StrictLevel::False;
        case core::StrictLevel::False:
        case core::StrictLevel::True:
        case core::StrictLevel::Strict:
            return level;

        case core::StrictLevel::Strong:
        case core::StrictLevel::Max:
            return core::StrictLevel::Strict;

        case core::StrictLevel::Internal:
        case core::StrictLevel::None:
        case core::StrictLevel::Autogenerated:
        case core::StrictLevel::Stdlib:
            Exception::raise("Should never attempt to recommend strict level={}", static_cast<uint8_t>(level));
    }
}

string levelToSigil(core::StrictLevel level) {
    switch (level) {
        case core::StrictLevel::Ignore:
            return "ignore";
        case core::StrictLevel::False:
            return "false";
        case core::StrictLevel::True:
            return "true";
        case core::StrictLevel::Strict:
            return "strict";
        case core::StrictLevel::Strong:
            return "strong";
        case core::StrictLevel::Stdlib:
            return "__STDLIB_INTERNAL";
        case core::StrictLevel::Max:
        case core::StrictLevel::Autogenerated:
        case core::StrictLevel::None:
        case core::StrictLevel::Internal:
            Exception::raise("Should never attempt to convert strict level={} to string", static_cast<uint8_t>(level));
    }
}

core::Loc findTyped(core::GlobalState &gs, core::FileRef file) {
    auto source = file.data(gs).source();

    if (file.data(gs).originalSigil == core::StrictLevel::None) {
        if (source.length() >= 2 && source[0] == '#' && source[1] == '!') {
            int newline = source.find("\n", 0);
            return core::Loc(file, newline + 1, newline + 1);
        }
        return core::Loc(file, 0, 0);
    }
    size_t start = 0;
    start = source.find("typed:", start);
    if (start == string_view::npos) {
        return core::Loc(file, 0, 0);
    }
    while (start >= 0 && source[start] != '#') {
        --start;
    }
    auto end = start;
    while (end < source.size() && source[end] != '\n') {
        ++end;
    }
    if (source[end] == '\n') {
        ++end;
    }
    return core::Loc(file, start, end);
}

void addInlineInput(const string &input, const string &filename, vector<core::FileRef> &inputFiles,
                    core::GlobalState &gs) {
    prodCounterAdd("types.input.bytes", input.size());
    prodCounterInc("types.input.lines");
    prodCounterInc("types.input.files");
    auto modifiedInput = input;
    if (core::File::fileStrictSigil(input) == core::StrictLevel::None) {
        // put it at the end so as to not upset line numbers
        modifiedInput += "\n# typed: true";
    }
    auto file = gs.enterFile(filename, modifiedInput);
    inputFiles.emplace_back(file);
}

#ifdef SORBET_REALMAIN_MIN

void runAutogen(const core::GlobalState &gs, options::Options &opts, WorkerPool &workers,
                vector<ast::ParsedFile> &indexed) {
    Exception::raise("Autogen is disabled in sorbet-orig for faster builds");
}

#else

struct AutogenResult {
    struct Serialized {
        // Selectively populated based on print options
        string strval;
        string msgpack;
        optional<autogen::Subclasses::Map> subclasses;
    };
    CounterState counters;
    vector<pair<int, Serialized>> prints;
};

void runAutogen(core::GlobalState &gs, options::Options &opts, WorkerPool &workers,
                std::unique_ptr<const OwnedKeyValueStore> kvstore) {
    if (!opts.inlineInput.empty()) {
        Exception::raise("Autogen ignores `-e`");
    }
    if (!opts.inlineRBIInput.empty()) {
        Exception::raise("Autogen ignores `--e-rbi`");
    }
    if (opts.cacheSensitiveOptions.sorbetPackages) {
        Exception::raise("Sorbet cannot run under `--sorbet-packages` when running for autogen");
    }
    if (opts.autocorrect) {
        Exception::raise("Autogen ignores `--autocorrect`");
    }

    auto inputFiles = pipeline::reserveFiles(gs, opts.inputFileNames);

    auto indexResult = pipeline::index(gs, inputFiles, opts, workers, kvstore);
    ENFORCE(indexResult.hasResult(), "There's no cancellation in autogen");
    auto indexed = move(indexResult.result());

    cache::maybeCacheGlobalStateAndFiles(OwnedKeyValueStore::abort(move(kvstore)), opts, gs, workers, indexed);

    // Only need to compute hashes when running to compute a FileHash
    auto foundHashes = nullptr;
    auto canceled = pipeline::name(gs, absl::MakeSpan(indexed), opts, workers, foundHashes);
    ENFORCE(!canceled, "There's no cancellation in batch mode");

    if (gs.hadCriticalError()) {
        gs.errorQueue->flushAllErrors(gs);
    }

    {
        core::UnfreezeNameTable nameTableAccess(gs);
        core::UnfreezeSymbolTable symbolAccess(gs);

        indexed = resolver::Resolver::runConstantResolution(gs, move(indexed), workers);
    }

    autogen::AutogenConfig autogenCfg = {
        .behaviorAllowedInRBIsPaths = std::move(opts.autogenBehaviorAllowedInRBIFilesPaths),
        .msgpackSkipReferenceMetadata = std::move(opts.autogenMsgpackSkipReferenceMetadata),
    };

    Timer timeit(logger, "autogen");

    auto resultq = make_shared<BlockingBoundedQueue<AutogenResult>>(indexed.size());
    auto fileq = make_shared<ConcurrentBoundedQueue<int>>(indexed.size());
    vector<AutogenResult::Serialized> merged(indexed.size());
    for (int i = 0; i < indexed.size(); ++i) {
        fileq->push(i, 1);
    }
    auto crcBuilder = autogen::CRCBuilder::create();
    int autogenVersion = opts.autogenVersion == 0 ? autogen::AutogenVersion::MAX_VERSION : opts.autogenVersion;

    workers.multiplexJob(
        "runAutogen", [&gs, &autogenVersion, &opts, &indexed, &autogenCfg, crcBuilder, fileq, resultq]() {
            AutogenResult out;
            int n = 0;
            {
                Timer timeit(logger, "autogenWorker");
                int idx = 0;

                for (auto result = fileq->try_pop(idx); !result.done(); result = fileq->try_pop(idx)) {
                    ++n;
                    auto &tree = indexed[idx];
                    // This does package-specific behavior without checking `--sorbet-packages`!
                    if (tree.file.data(gs).hasPackageRbPath()) {
                        continue;
                    }
                    if (autogenVersion < autogen::AutogenVersion::VERSION_INCLUDE_RBI && tree.file.data(gs).isRBI()) {
                        continue;
                    }

                    core::Context ctx(gs, core::Symbols::root(), tree.file);
                    auto leakTrees = true;
                    auto pf = autogen::Autogen::generate(ctx, move(tree), autogenCfg, *crcBuilder, leakTrees);
                    auto file = pf.file;

                    AutogenResult::Serialized serialized;

                    if (opts.print.Autogen.enabled) {
                        Timer timeit(logger, "autogenToString");
                        serialized.strval = pf.toString(ctx, autogenVersion);
                    }
                    if (opts.print.AutogenMsgPack.enabled) {
                        Timer timeit(logger, "autogenToMsgpack");
                        serialized.msgpack = pf.toMsgpack(ctx, autogenVersion, autogenCfg);
                    }

                    if (!file.data(gs).isRBI()) {
                        // Exclude RBI files because they are not loadable and should not appear in
                        // auto-loader related output.
                        if (opts.print.AutogenSubclasses.enabled) {
                            Timer timeit(logger, "autogenSubclasses");
                            serialized.subclasses = autogen::Subclasses::listAllSubclasses(
                                ctx, pf, opts.autogenSubclassesAbsoluteIgnorePatterns,
                                opts.autogenSubclassesRelativeIgnorePatterns);
                        }
                    }

                    out.prints.emplace_back(idx, move(serialized));
                }
            }

            out.counters = getAndClearThreadCounters();
            resultq->push(move(out), n);
        });

    AutogenResult out;
    for (auto res = resultq->wait_pop_timed(out, WorkerPool::BLOCK_INTERVAL(), *logger); !res.done();
         res = resultq->wait_pop_timed(out, WorkerPool::BLOCK_INTERVAL(), *logger)) {
        if (!res.gotItem()) {
            continue;
        }
        counterConsume(move(out.counters));
        for (auto &print : out.prints) {
            merged[print.first] = move(print.second);
        }
    }

    if (opts.print.Autogen.enabled || opts.print.AutogenMsgPack.enabled) {
        {
            Timer timeit(logger, "autogenDependencyDBPrint");
            if (opts.print.AutogenMsgPack.enabled) {
                opts.print.AutogenMsgPack.print(
                    autogen::ParsedFile::msgpackGlobalHeader(autogenVersion, merged.size(), autogenCfg));
            }
            for (auto &elem : merged) {
                if (opts.print.Autogen.enabled) {
                    opts.print.Autogen.print(elem.strval);
                }
                if (opts.print.AutogenMsgPack.enabled) {
                    opts.print.AutogenMsgPack.print(elem.msgpack);
                }
            }
        }
    }

    if (opts.print.AutogenSubclasses.enabled) {
        Timer timeit(logger, "autogenSubclassesPrint");

        // Merge the {Parent: Set{Child1, Child2}} maps from each thread
        autogen::Subclasses::Map childMap;
        for (const auto &el : merged) {
            if (!el.subclasses) {
                // File doesn't define any Child < Parent relationships
                continue;
            }

            for (const auto &[parentRef, children] : *el.subclasses) {
                auto &childEntry = childMap[parentRef];
                autogen::Subclasses::mergeInto(childEntry.entries, children.entries);
                childEntry.classKind = children.classKind;
            }
        }

        auto autogenSubclassesParentsRefs = vector<core::SymbolRef>();
        for (auto &parent : opts.autogenSubclassesParents) {
            auto parentRef = autogen::Subclasses::getConstantRef(gs, parent);
            if (!parentRef.exists())
                continue;
            autogenSubclassesParentsRefs.emplace_back(parentRef);
        }

        vector<string> serializedDescendantsMap =
            autogen::Subclasses::genDescendantsMap(gs, childMap, autogenSubclassesParentsRefs);

        opts.print.AutogenSubclasses.fmt("{}\n", fmt::join(serializedDescendantsMap, "\n"));
    }
}

#endif

int realmain(int argc, char *argv[]) {
#ifndef SORBET_REALMAIN_MIN
    initializeSymbolizer(argv[0]);
#endif
    returnCode = 0;
    logger = make_shared<spdlog::logger>("console", stderrColorSink);
    logger->set_level(spdlog::level::trace); // pass through everything, let the sinks decide
    logger->set_pattern("%v");
    fatalLogger = logger;

    auto typeErrorsConsole = make_shared<spdlog::logger>("typeDiagnostics", stderrColorSink);
    typeErrorsConsole->set_pattern("%v");

    auto extensionProviders = sorbet::pipeline::semantic_extension::SemanticExtensionProvider::getProviders();
    vector<unique_ptr<sorbet::pipeline::semantic_extension::SemanticExtension>> extensions;
    options::Options opts;
    options::readOptions(opts, extensions, argc, argv, extensionProviders, logger);
    if (opts.stdoutHUPHack) {
        startHUPMonitor();
    }
#ifndef SORBET_REALMAIN_MIN
    StatsD::addExtraTags(opts.metricsExtraTags);
#endif
    if (!opts.debugLogFile.empty()) {
        // LSP could run for a long time. Rotate log files, and trim at 1 GiB. Keep around 3 log files.
        // Cast first number to size_t to prevent integer multiplication.
        // TODO(jvilk): Reduce size once LSP logging is less chunderous.
        auto fileSink =
            make_shared<spdlog::sinks::rotating_file_sink_mt>(opts.debugLogFile, ((size_t)1) * 1024 * 1024 * 1024, 3);
        fileSink->set_pattern("timestamp=\"%Y-%m-%dT%T.%f\" level=%l %v");

        if (opts.logLevel >= 2) {
            fileSink->set_level(spdlog::level::trace);
        } else {
            fileSink->set_level(spdlog::level::debug);
        }
        { // replace console & fatal loggers
            vector<spdlog::sink_ptr> sinks{stderrColorSink, fileSink};
            auto combinedLogger = make_shared<spdlog::logger>("consoleAndFile", begin(sinks), end(sinks));
            combinedLogger->flush_on(spdlog::level::err);
            combinedLogger->set_level(spdlog::level::trace); // pass through everything, let the sinks decide

            spdlog::register_logger(combinedLogger);
            fatalLogger = combinedLogger;
            logger = combinedLogger;
        }
        { // replace type error logger
            vector<spdlog::sink_ptr> sinks{stderrColorSink, fileSink};
            auto combinedLogger = make_shared<spdlog::logger>("typeDiagnosticsAndFile", begin(sinks), end(sinks));
            spdlog::register_logger(combinedLogger);
            combinedLogger->set_level(spdlog::level::trace); // pass through everything, let the sinks decide
            typeErrorsConsole = combinedLogger;
        }
    }
    // Use a custom formatter so we don't get a default newline

    switch (opts.logLevel) {
        case 0:
            stderrColorSink->set_level(spdlog::level::info);
            break;
        case 1:
            stderrColorSink->set_level(spdlog::level::debug);
            logger->set_pattern("[T%t][%Y-%m-%dT%T.%f] %v");
            logger->debug("Debug logging enabled");
            break;
        default:
            stderrColorSink->set_level(spdlog::level::trace);
            logger->set_pattern("[T%t][%Y-%m-%dT%T.%f] %v");
            logger->trace("Trace logging enabled");
            break;
    }

    {
        string argsConcat(argv[0]);
        for (int i = 1; i < argc; i++) {
            absl::StrAppend(&argsConcat, " ", argv[i]);
        }
        logger->debug("Running sorbet version {} with arguments: {}", sorbet::full_version_string, argsConcat);
        if (!sorbet::is_release_build && !opts.silenceDevMessage &&
            std::getenv("SORBET_SILENCE_DEV_MESSAGE") == nullptr) {
            logger->info("👋 Hey there! Heads up that this is not a release build of sorbet.\n"
                         "Release builds are faster and more well-supported by the Sorbet team.\n"
                         "Check out the README to learn how to build Sorbet in release mode.\n"
                         "To forcibly silence this error, either pass --silence-dev-message,\n"
                         "or set SORBET_SILENCE_DEV_MESSAGE=1 in your shell environment.\n");
        }
    }

    prodCounterSet("worker_pool_size", opts.threads);
    unique_ptr<WorkerPool> workers = WorkerPool::create(opts.threads, *logger);

    auto errorFlusher = make_shared<core::ErrorFlusherStdout>();
    unique_ptr<core::GlobalState> gs =
        make_unique<core::GlobalState>(make_shared<core::ErrorQueue>(*typeErrorsConsole, *logger, errorFlusher));

    logger->trace("cleaning up old state");
    cache::SessionCache::reapOldCaches(opts);

    logger->trace("building initial global state");

    unique_ptr<const OwnedKeyValueStore> kvstore = cache::maybeCreateKeyValueStore(logger, opts);
    payload::createInitialGlobalState(*gs, opts, kvstore);
    pipeline::setGlobalStateOptions(*gs, opts);

    // This is here, not in setGlobalStateOptions, because this makes us allocate memory, potentially lots of it.
    // We want to be able to use setGlobalStateOptions in places like makeEmptyGlobalStateForFile,
    // which will only ever need enough memory for one file's worth of definition (not one codebase's worth).
    gs->preallocateTables(opts.reserveClassTableCapacity, opts.reserveMethodTableCapacity,
                          opts.reserveFieldTableCapacity, opts.reserveTypeParameterTableCapacity,
                          opts.reserveTypeMemberTableCapacity, opts.reserveUtf8NameTableCapacity,
                          opts.reserveConstantNameTableCapacity, opts.reserveUniqueNameTableCapacity);

    if (opts.cacheSensitiveOptions.runningUnderAutogen) {
        gs->cacheSensitiveOptions.runningUnderAutogen = true;
    }

    if (gs->cacheSensitiveOptions.runningUnderAutogen) {
        gs->suppressErrorClass(core::errors::Namer::RedefinitionOfMethod.code);
        gs->suppressErrorClass(core::errors::Namer::ModuleKindRedefinition.code);
        gs->suppressErrorClass(core::errors::Namer::ConstantKindRedefinition.code);
        gs->suppressErrorClass(core::errors::Resolver::StubConstant.code);
        gs->suppressErrorClass(core::errors::Resolver::RecursiveTypeAlias.code);
        gs->suppressErrorClass(core::errors::Resolver::AmbiguousDefinitionError.code);
    }

    gs->semanticExtensions = move(extensions);

    logger->trace("done building initial global state");

    if (opts.print.PayloadSources.enabled) {
        auto dumpDir = opts.print.PayloadSources.outputPath;
        FileOps::ensureDir(dumpDir);

        // skip idx 0 (corresponds to File that does not exist, so it contains nullptr)
        for (auto &payloadFile : gs->getFiles().subspan(1)) {
            auto payloadVersion = sorbet::is_release_build ? sorbet::build_scm_revision : "master";
            auto payloadPath = payloadFile->path();
            auto payloadPrefix = absl::StrCat("https://github.com/sorbet/sorbet/tree/", payloadVersion, "/rbi/");

            if (!absl::StartsWith(payloadPath, payloadPrefix)) {
                // Skip files from `bazel-out/`
                continue;
            }

            payloadPath.remove_prefix(payloadPrefix.size());

            vector<string_view> parts = absl::StrSplit(payloadPath, "/");
            auto dumpSubdir = dumpDir;
            for (int i = 0; i < parts.size() - 1; i++) {
                auto part = parts[i];
                dumpSubdir = absl::StrCat(dumpSubdir, "/", part);
                FileOps::ensureDir(dumpSubdir);
            }

            auto dumpPath = absl::StrCat(dumpDir, "/", payloadPath);
            opts.fs->writeFile(dumpPath, payloadFile->source());
        }

        return returnCode;
    }

    if (opts.runLSP) {
#ifdef SORBET_REALMAIN_MIN
        logger->warn("LSP is disabled in sorbet-orig for faster builds");
        return 1;
#else
        logger->debug("Starting sorbet version {} in LSP server mode. "
                      "Talk ‘\\r\\n’-separated JSON-RPC to me. "
                      "More details at https://microsoft.github.io/language-server-protocol/specification."
                      "If you're developing an LSP extension to some editor, make sure to run sorbet with `-v` flag,"
                      "it will enable outputting the LSP session to stderr(`Write: ` and `Read: ` log lines)",
                      sorbet::full_version_string);

        auto output = make_shared<lsp::LSPStdout>(logger);
        lsp::LSPLoop loop(move(gs), *workers, make_shared<lsp::LSPConfiguration>(opts, output, logger),
                          OwnedKeyValueStore::abort(move(kvstore)));
        gs = loop.runLSP(make_shared<lsp::LSPFDInput>(logger, STDIN_FILENO)).value_or(nullptr);
    } else if (gs->cacheSensitiveOptions.runningUnderAutogen) {
        Timer timeall(logger, "wall_time");
        runAutogen(*gs, opts, *workers, move(kvstore));

        gs->errorQueue->flushAllErrors(*gs);
        if (!opts.noErrorCount) {
            errorFlusher->flushErrorCount(gs->errorQueue->logger, gs->errorQueue->nonSilencedErrorCount);
        }
#endif
    } else {
        Timer timeall(logger, "wall_time");
        if constexpr (sorbet::is_release_build) {
            if (opts.statsdTagRevision) {
                timeall.setTag("version", sorbet::build_scm_revision);
            }
        }

        vector<core::FileRef> inputFiles;
        logger->trace("Files: ");

        if (!opts.storeState.empty()) {
            // Compute file hashes for payload files (which aren't part of inputFiles) for LSP
            hashing::Hashing::computeFileHashes(gs->getFiles(), *logger, *workers, opts);
        }

        inputFiles = pipeline::reserveFiles(*gs, opts.inputFileNames);

        // We explicitly free the input names here, as we won't use them for the remainder of execution, and on large
        // codebases they take up a non-trivial amount of memory.
        opts.inputFileNames = vector<string>();

        {
            core::UnfreezeFileTable fileTableAccess(*gs);
            if (!opts.inlineInput.empty()) {
                addInlineInput(opts.inlineInput, "-e", inputFiles, *gs);
            }
            if (!opts.inlineRBIInput.empty()) {
                addInlineInput(opts.inlineRBIInput, "--e-rbi.rbi", inputFiles, *gs);
            }
        }

        auto inputFilesSpan = absl::Span<core::FileRef>(inputFiles);

        // ----- build the package DB -----

        vector<ast::ParsedFile> packageIndexed;
        auto inputPackageFiles = absl::Span<core::FileRef>{};
        if (opts.cacheSensitiveOptions.sorbetPackages) {
            auto numPackageFiles = pipeline::partitionPackageFiles(*gs, inputFilesSpan);
            inputPackageFiles = inputFilesSpan.first(numPackageFiles);
            inputFilesSpan = inputFilesSpan.subspan(numPackageFiles);

            if (!opts.storeState.empty() || opts.forceHashing) {
                auto result = hashing::Hashing::indexAndComputeFileHashes(*gs, opts, *logger, inputPackageFiles,
                                                                          *workers, kvstore);
                ENFORCE(result.hasResult(), "There's no cancellation in batch mode");
                packageIndexed = std::move(result.result());
            } else {
                auto result = pipeline::index(*gs, inputPackageFiles, opts, *workers, kvstore);
                ENFORCE(result.hasResult(), "There's no cancellation in batch mode");
                packageIndexed = std::move(result.result());
            }

            // Cache these before any packager rewrites, so that the cache is still
            // usable regardless of whether `--sorbet-packages` was passed.
            // Want to keep the kvstore around so we can still write to it later.
            kvstore = cache::ownIfUnchanged(
                *gs, cache::maybeCacheGlobalStateAndFiles(OwnedKeyValueStore::abort(move(kvstore)), opts, *gs, *workers,
                                                          packageIndexed));

            // Populate the packageDB by processing only the __package.rb files.
            // Only need to compute hashes when running to compute a FileHash
            auto foundHashes = nullptr;
            auto canceled =
                pipeline::name(*gs, absl::Span<ast::ParsedFile>(packageIndexed), opts, *workers, foundHashes);
            ENFORCE(!canceled, "There's no cancellation in batch mode");
            pipeline::buildPackageDB(*gs, absl::MakeSpan(packageIndexed), inputFilesSpan, opts, *workers);
        }

        // We disable tree leaking if we're targeting emscripten, or if we're typechecking in package dependency order.
        // The latter is so that we reuse memory as we traverse the package graph.
        const bool intentionallyLeakASTs = !sorbet::emscripten_build && !opts.packageDirected;

        // The rest of the pipeline proceeds by strata in the package condensation graph. When stripe-packages is not
        // enabled, everything ends up in one big stratum.
        vector<ast::ParsedFile> stratumFiles;
        int currentStratum = -1;
        auto strata = pipeline::computePackageStrata(*gs, packageIndexed, inputFilesSpan, opts);
        gs->preallocateForStrata(strata.strata.size());
        for (auto &stratum : strata.strata) {
            ++currentStratum;

            // We can unconditionally reset (to drop the vectors) instead of having to consult
            // intentionallyLeakASTs, because if intentionallyLeakASTs, then necessarily
            // packageDirected will be false, and thus this loop will only iterate once, and since
            // it's at the beginning, the list will be empty to start, and thus a no-op.
            Parallel::iterate(*workers, "clearStratumFiles", absl::MakeSpan(stratumFiles), [](auto &job) {
                // Force the destructor of `ast::ExpressionPtr` to run for `job.tree`.
                job.tree.reset();
            });
            stratumFiles.clear();
            stratumFiles.reserve(stratum.packageFiles.size() + stratum.sourceFiles.size());

            absl::c_move(stratum.packageFiles, back_inserter(stratumFiles));

            {
                // ----- index -----

                auto nonPackageIndexedResult =
                    (!opts.storeState.empty() || opts.forceHashing)
                        // Calculate file hashes alongside indexing when --store-state is specified for LSP mode
                        ? hashing::Hashing::indexAndComputeFileHashes(*gs, opts, *logger, stratum.sourceFiles, *workers,
                                                                      kvstore)
                        : pipeline::index(*gs, stratum.sourceFiles, opts, *workers, kvstore);
                ENFORCE(nonPackageIndexedResult.hasResult(), "There's no cancellation in batch mode");
                auto nonPackageIndexed = std::move(nonPackageIndexedResult.result());

                // Cache these before any packager rewrites, so that the cache is still usable
                // regardless of whether `--sorbet-packages` was passed.
                kvstore = cache::ownIfUnchanged(
                    *gs, cache::maybeCacheGlobalStateAndFiles(OwnedKeyValueStore::abort(move(kvstore)), opts, *gs,
                                                              *workers, nonPackageIndexed));

                // If we're not running in package-directed mode, there will only be a single iteration of this loop and
                // we can safely free the cache early.
                if (!opts.packageDirected) {
                    kvstore.reset();
                }

                // Only need to compute hashes when running to compute a FileHash
                auto foundHashes = nullptr;
                auto canceled = pipeline::name(*gs, absl::MakeSpan(nonPackageIndexed), opts, *workers, foundHashes);
                ENFORCE(!canceled, "There's no cancellation in batch mode");
                // Now validate all the other files (the packageDB shouldn't change)
                pipeline::validatePackagedFiles(*gs, absl::MakeSpan(nonPackageIndexed), opts, *workers);

                pipeline::unpartitionPackageFiles(stratumFiles, move(nonPackageIndexed));

                if (gs->hadCriticalError()) {
                    gs->errorQueue->flushAllErrors(*gs);
                }
            }

            {
                stratumFiles = move(pipeline::resolve(*gs, move(stratumFiles), opts, *workers).result());
                if (gs->hadCriticalError()) {
                    gs->errorQueue->flushAllErrors(*gs);
                }

                if (opts.genPackagesMode == core::packages::GenPackagesMode::Disabled) {
                    // In --gen-packages mode, we skip typecheck because we only want to show packaging related errors,
                    // and skipping typecheck saves a significant amount of time.
                    pipeline::sortBySize(*gs, stratumFiles);
                    pipeline::typecheck(*gs, move(stratumFiles), opts, *workers, /* cancelable */ false,
                                        core::packages::Stratum(currentStratum),
                                        /* preemptionManager */ nullptr, intentionallyLeakASTs);

                    if (gs->hadCriticalError()) {
                        gs->errorQueue->flushAllErrors(*gs);
                    }
                }
            }
        }

        if (opts.genPackagesMode != core::packages::GenPackagesMode::Disabled) {
            // One of the things this pass does is insert missing exports. Because of this, it needs to run after
            // VisibilityChecker has been run for all strata; a symbol might be used in a strata after the strata for
            // the package that owns that symbol, and we need to be able to see that use to know that a export should be
            // generated for that symbol. Because of that, we need to put this pass outside of the loop above.
            //
            // A similar principle applies for inserting `visible_to`s with --gen-packages-update-visiblity-for
            switch (opts.genPackagesMode) {
                case core::packages::GenPackagesMode::Strict:
                    packager::GenPackages::runStrict(*gs);
                    break;
                case core::packages::GenPackagesMode::Normal:
                    packager::GenPackages::run(*gs);
                    break;
                case core::packages::GenPackagesMode::Disabled:
                    ENFORCE(false);
            }

            // One thing typecheck does is call flushErrorsForFile, which provides a consistent
            // ordering for errors when running in single threaded mode. To replicate that behaviour, we loop
            // over all files and call it manually here, instead of waiting for the flushAllErrors call later.
            for (auto &fileRef : inputPackageFiles) {
                gs->errorQueue->flushErrorsForFile(*gs, fileRef);
            }

            for (auto &fileRef : inputFilesSpan) {
                gs->errorQueue->flushErrorsForFile(*gs, fileRef);
            }

            if (intentionallyLeakASTs) {
                // All trees get free'd or leaked during `pipeline::typecheck` which is skipped when we're in
                // genPackages mode. Leak them here if we're going to.
                for (auto &e : packageIndexed) {
                    intentionallyLeakMemory(e.tree.release());
                }
            }
        }

        // getAndClearHistogram ensures that we don't accidentally submit a high-cardinality histogram to statsd
        auto untypedUsages = getAndClearHistogram("untyped.usages");
        pipeline::printGlobalTables(*gs, opts, untypedUsages);

        if (!opts.minimizeRBI.empty()) {
#ifdef SORBET_REALMAIN_MIN
            logger->warn("--minimize-rbi is disabled in sorbet-orig for faster builds");
            return 1;
#else
            // In the future, we might consider making minimizeRBI be a repeatable option, and run
            // this block once for each input file.
            // The trick there is that they would all currently output to the same file, even for
            // multiple input files if we assume the naive implementation, which might not be the
            // API we want to expose.

            auto optsForMinimize = opts.clone();
            // Explicitly turn off the packager, because it doesn't make sense when the whole
            // project is a single RBI file.
            optsForMinimize.cacheSensitiveOptions.sorbetPackages = false;

            unique_ptr<core::GlobalState> gsForMinimize = make_unique<core::GlobalState>(gs->errorQueue);
            auto kvstore = nullptr;
            payload::createInitialGlobalState(*gsForMinimize, optsForMinimize, kvstore);
            pipeline::setGlobalStateOptions(*gsForMinimize, optsForMinimize);

            Minimize::indexAndResolveForMinimize(*gs, *gsForMinimize, optsForMinimize, *workers, opts.minimizeRBI);
            UnorderedMap<long, long> untypedUsagesForMinimize;
            // Use `opts` not `optsForMinimize` so that the print outputs go to the same spot.
            pipeline::printGlobalTables(*gsForMinimize, opts, untypedUsagesForMinimize);
            Minimize::writeDiff(*gs, *gsForMinimize, opts.print.MinimizeRBI);

            if (gsForMinimize->hadCriticalError()) {
                returnCode = 10;
            }

            intentionallyLeakMemory(gsForMinimize.release());
#endif
        }

        if (opts.suggestTyped) {
#ifdef SORBET_REALMAIN_MIN
            logger->warn("Signature suggestion is disabled in sorbet-orig for faster builds");
            return 1;
#else
            auto id = 0;
            for (auto &file : gs->getFiles().subspan(1)) {
                id++;
                if (file->isPayload()) {
                    continue;
                }
                if (file->minErrorLevel() <= core::StrictLevel::Ignore) {
                    continue;
                }
                if (file->originalSigil > core::StrictLevel::Max) {
                    // don't change the sigil on "special" files
                    continue;
                }
                auto minErrorLevel = levelMinusOne(file->minErrorLevel());
                if (file->originalSigil == minErrorLevel) {
                    continue;
                }
                minErrorLevel = levelToRecommendation(minErrorLevel);
                if (file->originalSigil == minErrorLevel) {
                    // if the file could be strong, but is only marked strict, ensure that we don't recommend that it be
                    // marked strict.
                    continue;
                }
                auto fref = core::FileRef(id);
                auto loc = findTyped(*gs, fref);
                if (auto e = gs->beginError(loc, core::errors::Infer::SuggestTyped)) {
                    auto sigil = levelToSigil(minErrorLevel);
                    e.setHeader("You could add `# typed: {}`", sigil);
                    e.replaceWith(fmt::format("Add `typed: {}` sigil", sigil), loc, "# typed: {}\n", sigil);
                }
            }
#endif
        }

        if (!opts.storeState.empty()) {
            // It's possible that we want to move this to GlobalPass and provide some way for users
            // to opt into this check in their own codebases. Punting on that for now.
            for (auto i = 1; i < gs->classAndModulesUsed(); i++) {
                auto sym = core::ClassOrModuleRef(*gs, i);
                auto data = sym.data(*gs);
                auto loc = data->loc();
                auto file = loc.file();
                if (file.exists() && file.data(*gs).isPayload() && !data->isDeclared()) {
                    if (data->isSingletonClass(*gs) && data->attachedClass(*gs).data(*gs)->isDeclared()) {
                        // Have to treat singleton classes specially, because when they're created
                        // synthetically they are not declared, but that's ~fine as long the
                        // corresponding attached class is declared.
                        continue;
                    }

                    if (auto e = gs->beginError(loc, core::errors::Internal::InternalError)) {
                        // This usually represents a typo/omission.
                        // It's best to figure out how it was defined in the stdlib and capture that.
                        e.setHeader("Refusing to implicitly declare `{}` as a module in the payload", sym.show(*gs));
                    }
                }
            }
        }

        gs->errorQueue->flushAllErrors(*gs);

        if (!opts.noErrorCount) {
            errorFlusher->flushErrorCount(gs->errorQueue->logger, gs->errorQueue->nonSilencedErrorCount);
        }
        if (opts.autocorrect) {
            errorFlusher->flushAutocorrects(*gs, *opts.fs);
        }
        logger->trace("sorbet done");

        if (!opts.storeState.empty()) {
            ENFORCE(opts.storeState.size() == 3);
            gs->markAsPayload();
            auto result = core::serialize::Serializer::store(*gs);
            FileOps::write(opts.storeState[0].c_str(), result.symbolTableData);
            FileOps::write(opts.storeState[1].c_str(), result.nameTableData);
            FileOps::write(opts.storeState[2].c_str(), result.fileTableData);
        }

        auto untypedBlames = getAndClearHistogram("untyped.blames");
        if constexpr (sorbet::track_untyped_blame_mode) {
            pipeline::printUntypedBlames(*gs, untypedBlames, opts);
        }
    }

#ifdef SORBET_REALMAIN_MIN
    if (opts.enableCounters || !opts.statsdHost.empty() || !opts.webTraceFile.empty() || !opts.metricsFile.empty()) {
        logger->warn("Metrics are disabled in sorbet-orig for faster builds");
        return 1;
    }
#else
    StatsD::addStandardMetrics();

    if (opts.enableCounters) {
        logger->warn("" + getCounterStatistics());
    }

    auto counters = getAndClearThreadCounters();

    if (!opts.statsdHost.empty()) {
        auto prefix = opts.statsdPrefix;
        if (opts.runLSP) {
            prefix += ".lsp";
        }
        StatsD::submitCounters(counters, opts.statsdHost, opts.statsdPort, prefix + ".counters");
    }
    if (!opts.webTraceFile.empty()) {
        web_tracer_framework::Tracing::storeTraces(counters, opts.webTraceFile, opts.webTraceFileStrict);
    }

    if (!opts.metricsFile.empty()) {
        string status;
        if (gs->hadCriticalError()) {
            status = "Error";
        } else if (returnCode != 0) {
            status = "Failure";
        } else {
            status = "Success";
        }

        auto json = core::JSON::metricsToJSON(counters, opts.metricsPrefix, opts.metricsRepo, opts.metricsBranch,
                                              opts.metricsSha, status);

        try {
            opts.fs->writeFile(opts.metricsFile, json);
        } catch (FileNotFoundException e) {
            logger->error("Cannot write metrics file at `{}`", opts.metricsFile);
        }
    }
#endif
    if (!gs || gs->hadCriticalError()) {
        returnCode = 10;
    } else if (returnCode == 0 && gs->totalErrors() > 0 && !opts.suppressNonCriticalErrors) {
        returnCode = 100;
    }

    opts.flushPrinters();

    if (!sorbet::emscripten_build) {
        intentionallyLeakMemory(gs.release());
    }

    // je_malloc_stats_print(nullptr, nullptr, nullptr); // uncomment this to print jemalloc statistics

    return returnCode;
}

} // namespace sorbet::realmain
