/*
 * Copyright (c) 2012 Tim Ruehsen
 * Copyright (c) 2015-2026 Free Software Foundation, Inc.
 *
 * This file is part of libwget.
 *
 * Libwget is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Libwget is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public License
 * along with libwget.  If not, see <https://www.gnu.org/licenses/>.
 *
 *
 * routines to parse robots.txt (RFC 9309)
 *
 * Changelog
 * 28.09.2013  Tim Ruehsen  created
 * 15.03.2024  Avinash Sonawane, Tim Ruehsen  updated to RFC 9309 (except for the Allow field)
 *
 */

#include <config.h>

#include <string.h>
#include <ctype.h>
#include <stdbool.h>

#include <wget.h>
#include "private.h"

/**
 * \file
 * \brief Robots Exclusion file parser
 * \defgroup libwget-robots Robots Exclusion file parser
 * @{
 *
 * The purpose of this set of functions is to parse a
 * Robots Exclusion Standard file into a data structure
 * for easy access.
 */

struct wget_robots_st {
	wget_vector
		*paths;    //!< paths found in robots.txt (element: wget_string)
	wget_vector
		*sitemaps; //!< sitemaps found in robots.txt (element: char *)
};

static void path_free(void *path)
{
	wget_string *p = path;

	xfree(p->p);
	xfree(p);
}

static inline void advance_ws(const char **s)
{
	for (; isblank(**s); (*s)++);
}

static bool parse_record_field(const char **data, const char *field, size_t field_length)
{
	advance_ws(data);

	if (wget_strncasecmp_ascii(*data, field, field_length))
		return false;

	*data += field_length;
	advance_ws(data);

	if (**data != ':')
		return false;

	*data += 1;
	advance_ws(data);

	return true;
}
#define parse_record_field(d, f) parse_record_field(d, f, sizeof(f) - 1)


/**
 * \param[in] data Memory with robots.txt content (with trailing 0-byte)
 * \param[in] client Name of the client / user-agent
 * \return Return an allocated wget_robots structure or NULL on error
 *
 * The function parses the robots.txt \p data in accordance to
 * https://www.robotstxt.org/orig.html#format and returns a ROBOTS structure
 * including a list of the disallowed paths and including a list of the sitemap
 * files.
 *
 * The ROBOTS structure has to be freed by calling wget_robots_free().
 */
int wget_robots_parse(wget_robots **_robots, const char *data, const char *client)
{
	wget_robots *robots;
	wget_string path;
	size_t client_length = client ? strlen(client) : 0;
	const char *p;
	bool seek_record_client = false;
	enum record {
		NOT_IN_RECORD,
		/* User-agent:client */
		IN_RECORD_CLIENT,
		/* User-agent:* */
		IN_RECORD_STAR,
		/* Disallow:foo */
		ADDED_DISALLOW,
		NO_MORE_RECORDS
	} state;

	if (!data || !*data || !_robots)
		return WGET_E_INVALID;

	if (!(robots = wget_calloc(1, sizeof(wget_robots))))
		return WGET_E_MEMORY;

	state = NOT_IN_RECORD;
	do {
		if (state != NO_MORE_RECORDS && state != IN_RECORD_CLIENT && parse_record_field(&data, "User-agent")) {
			if (client && !wget_strncasecmp_ascii(data, client, client_length)) {
				if (!seek_record_client)
					wget_vector_free(&robots->paths);
				seek_record_client = true;
				state = IN_RECORD_CLIENT;
			} else if (!seek_record_client && (*data == '*'))
				state = IN_RECORD_STAR;
			else if (state == ADDED_DISALLOW)
				state = NOT_IN_RECORD;
		} else if (state != NO_MORE_RECORDS && state != NOT_IN_RECORD && parse_record_field(&data, "Disallow")) {
			if (!*data || isspace(*data) || *data == '#') {
				// all allowed
				wget_vector_free(&robots->paths);
				if (seek_record_client)
					state = NO_MORE_RECORDS;
				else {
					state = NOT_IN_RECORD;
					seek_record_client = true;
				}
			} else {
				if (!robots->paths) {
					if (!(robots->paths = wget_vector_create(32, NULL)))
						goto oom;
					wget_vector_set_destructor(robots->paths, path_free);
				}
				for (p = data; *p && !isspace(*p) && *p != '#'; p++);
				path.len = p - data;
				if (!(path.p = wget_strmemdup(data, path.len)))
					goto oom;
				if (wget_vector_add_memdup(robots->paths, &path, sizeof(path)) < 0) {
					xfree(path.p);
					goto oom;
				}
				state = ADDED_DISALLOW;
			}
		} else if (parse_record_field(&data, "Sitemap")) {
			for (p = data; *p && !isspace(*p) && *p != '#'; p++);

			if (p > data){
				if (!robots->sitemaps)
					if (!(robots->sitemaps = wget_vector_create(4, NULL)))
						goto oom;

				char *sitemap = wget_strmemdup(data, p - data);
				if (!sitemap)
					goto oom;
				if (wget_vector_add(robots->sitemaps, sitemap) < 0)
					goto oom;
			}
		}

		if ((data = strchr(data, '\n')))
			data++; // point to next line
	} while (data && *data);

/*
	for (int it = 0; it < wget_vector_size(robots->paths); it++) {
		ROBOTS_PATH *path = wget_vector_get(robots->paths, it);
		debug_printf("path '%s'\n", path->path);
	}
	for (int it = 0; it < wget_vector_size(robots->sitemaps); it++) {
		const char *sitemap = wget_vector_get(robots->sitemaps, it);
		debug_printf("sitemap '%s'\n", sitemap);
	}
*/

	*(_robots) = robots;
	return WGET_E_SUCCESS;

oom:
	wget_robots_free(&robots);
	return WGET_E_MEMORY;
}

/**
 * \param[in,out] robots Pointer to Pointer to wget_robots structure
 *
 * wget_robots_free() free's the formerly allocated wget_robots structure.
 */
void wget_robots_free(wget_robots **robots)
{
	if (robots && *robots) {
		wget_vector_free(&(*robots)->paths);
		wget_vector_free(&(*robots)->sitemaps);
		xfree(*robots);
		*robots = NULL;
	}
}

/**
 * @param robots Pointer to instance of wget_robots
 * @return Returns the number of paths listed in \p robots
 */
int wget_robots_get_path_count(wget_robots *robots)
{
	if (robots)
		return wget_vector_size(robots->paths);

	return 0;
}

/**
 * @param robots Pointer to instance of wget_robots
 * @param index Index of the wanted path
 * @return Returns the path at \p index or NULL
 */
wget_string *wget_robots_get_path(wget_robots *robots, int index)
{
	if (robots && robots->paths)
		return wget_vector_get(robots->paths, index);

	return NULL;
}

/**
 * @param robots Pointer to instance of wget_robots
 * @return Returns the number of sitemaps listed in \p robots
 */
int wget_robots_get_sitemap_count(wget_robots *robots)
{
	if (robots)
		return wget_vector_size(robots->sitemaps);

	return 0;
}

/**
 * @param robots Pointer to instance of wget_robots
 * @param index Index of the wanted sitemap URL
 * @return Returns the sitemap URL at \p index or NULL
 */
const char *wget_robots_get_sitemap(wget_robots *robots, int index)
{
	if (robots && robots->sitemaps)
		return wget_vector_get(robots->sitemaps, index);

	return NULL;
}

/**@}*/
