src/parser.c
Rama master · réplica de lectura del repositorio original.
/**
* parser.c -- web log parsing
* ______ ___
* / ____/___ / | _____________ __________
* / / __/ __ \/ /| |/ ___/ ___/ _ \/ ___/ ___/
* / /_/ / /_/ / ___ / /__/ /__/ __(__ |__ )
* \____/\____/_/ |_\___/\___/\___/____/____/
*
* The MIT License (MIT)
* Copyright (c) 2009-2026 Gerardo Orellana <hello @ goaccess.io>
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
/*
* "_XOPEN_SOURCE" is required for the GNU libc to export "strptime(3)"
* correctly.
*/
#define _LARGEFILE_SOURCE
#define _LARGEFILE64_SOURCE
#define _FILE_OFFSET_BITS 64
#define _XOPEN_SOURCE 700
#define _DEFAULT_SOURCE
#include <ctype.h>
#include <errno.h>
#if HAVE_CONFIG_H
#include <config.h>
#endif
#include <stddef.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <strings.h>
#include <sys/stat.h>
#include <unistd.h>
#include <fcntl.h>
#include <inttypes.h>
#include <limits.h>
#include <pthread.h>
#include <libgen.h>
#include "gkhash.h"
#include "parser.h"
#include "browsers.h"
#include "error.h"
#include "goaccess.h"
#include "gstorage.h"
#include "util.h"
#include "websocket.h"
#include "xmalloc.h"
/* Conventional marker emitted when an optional log field has no value. */
#define MISSING_LOG_VALUE "-"
typedef struct GCacheStatusRule_ {
const char *value, *status;
int prefix; /* Apache appends details only to some status messages. */
} GCacheStatusRule;
/* TLS wire-protocol version codes emitted by Caddy's JSON access logger. */
enum GTLSVersionCode {
TLS_VERSION_SSL3 = 0x0300,
TLS_VERSION_1_0 = 0x0301,
TLS_VERSION_1_1 = 0x0302,
TLS_VERSION_1_2 = 0x0303,
TLS_VERSION_1_3 = 0x0304,
};
/* Allocate memory for a new GRawData instance.
*
* On success, the newly allocated GRawData is returned . */
GRawData *
new_grawdata (void) {
GRawData *raw_data = xmalloc (sizeof (*raw_data));
memset (raw_data, 0, sizeof *raw_data);
return raw_data;
}
/* Allocate memory for a new GRawDataItem instance.
*
* On success, the newly allocated GRawDataItem is returned . */
GRawDataItem *
new_grawdata_item (unsigned int size) {
GRawDataItem *item = xcalloc (size, sizeof (*item));
return item;
}
/* Free memory allocated for a GRawData and GRawDataItem instance. */
void
free_raw_data (GRawData *raw_data) {
free (raw_data->items);
free (raw_data);
}
/* Reset an instance of GLog structure. */
void
reset_struct (Logs *logs) {
int i = 0;
for (i = 0; i < logs->size; ++i)
logs->glog[i].invalid = logs->glog[i].processed = 0;
}
/* Allocate memory for a new Logs and GLog instance.
*
* On success, the newly allocated Logs is returned . */
Logs *
new_logs (int size) {
Logs *logs = xmalloc (sizeof (*logs));
memset (logs, 0, sizeof *logs);
logs->glog = xcalloc (size, sizeof (GLog));
logs->size = size;
logs->idx = 0;
return logs;
}
/* Allocate, initialize and add the given filename to our logs structure.
*
* On error, 1 is returned.
* On success, the given filename is added to the Logs structure and 0 is
* returned. */
int
set_glog (Logs *logs, const char *filename) {
GLog *tmp = NULL, *glog = NULL;
int newlen = 0;
char const *err;
char *fvh = NULL, *fn = NULL, *temp_path = NULL;
char resolved[PATH_MAX] = { 0 };
const char *final_path = NULL;
if (logs->size - 1 < logs->idx) {
newlen = logs->size + 1;
if (!(tmp = xrealloc (logs->glog, newlen * sizeof (GLog))))
return ERR_LOG_REALLOC_FAILURE;
logs->glog = tmp;
memset (logs->glog + logs->idx, 0, (logs->idx + 1 - logs->size) * sizeof *logs->glog);
logs->size = newlen;
}
fn = xstrdup (filename); /* ensure fn is a string */
glog = logs->glog;
glog[logs->idx].errors = xcalloc (MAX_LOG_ERRORS, sizeof (char *));
/* Resolve to absolute path for clearer display in spinners.
* realpath() returns NULL if file doesn't exist or realpath fails; fall back to original. */
final_path = fn;
if (realpath (fn, resolved) != NULL) {
final_path = resolved;
}
glog[logs->idx].props.filename = xstrdup (final_path);
/* Create temporary copy for basename() since it expects non-const argument. */
temp_path = xstrdup (final_path);
glog[logs->idx].props.fname = xstrdup (basename (temp_path));
free (temp_path);
if (!glog->pipe && conf.fname_as_vhost) {
if (!(fvh = regex_extract_string (glog[logs->idx].props.fname, conf.fname_as_vhost, 1, &err)))
FATAL ("%s %s[%s]", err, glog[logs->idx].props.fname, conf.fname_as_vhost);
glog[logs->idx].fname_as_vhost = fvh;
}
logs->processed = &(glog[logs->idx].processed);
logs->filename = glog[logs->idx].props.filename;
logs->idx++;
free (fn);
return 0;
}
/* Ensure the given filename is part of our original list of files.
*
* On error, 1 is returned.
* On success, the given filename is added to the Logs structure and 0 is
* returned. */
int
set_log (Logs *logs, const char *value) {
if (str_inarray (value, conf.filenames, conf.filenames_idx) < 0)
return ERR_LOG_NOT_FOUND;
return set_glog (logs, value);
}
/* Allocate memory for a new set of Logs including a GLog instance.
*
* On success, the newly allocated Logs is returned . */
Logs *
init_logs (int size) {
Logs *logs = NULL;
GLog *glog = NULL;
int i = 0, ret = 0;
/* if no logs no a pipe nor restoring, nothing to do then */
if (!size && !conf.restore)
return NULL;
/* If no logs nor a pipe but restoring, we still need an minimal instance of
* logs and a glog */
if (!size) {
logs = xcalloc (1, sizeof (*logs));
logs->glog = xcalloc (1, sizeof (*glog));
logs->processed = &(logs->glog[0].processed);
return logs;
}
logs = new_logs (size);
logs->size = size;
for (i = 0; i < size; ++i) {
if ((ret = set_log (logs, conf.filenames[i])))
FATAL ("%s\n", ERR_LOG_NOT_FOUND_MSG);
}
return logs;
}
/* Free all log errors stored during parsing. */
void
free_logerrors (GLog *glog) {
int i;
if (!glog->log_erridx)
return;
for (i = 0; i < glog->log_erridx; ++i)
free (glog->errors[i]);
glog->log_erridx = 0;
}
/* Free all log containers. */
void
free_logs (Logs *logs) {
GLog *glog = NULL;
int i;
for (i = 0; i < logs->size; ++i) {
glog = &logs->glog[i];
free (glog->props.filename);
free (glog->props.fname);
free (glog->fname_as_vhost);
free_logerrors (glog);
free (glog->errors);
pthread_mutex_destroy (&glog->error_mutex); // Destroy mutex
if (glog->pipe) {
fclose (glog->pipe);
}
}
free (logs->glog);
free (logs);
}
/* Initialize a new GLogItem instance.
*
* On success, the new GLogItem instance is returned. */
GLogItem *
init_log_item (GLog *glog) {
GLogItem *logitem;
logitem = xmalloc (sizeof (GLogItem));
memset (logitem, 0, sizeof *logitem);
logitem->agent = NULL;
logitem->browser = NULL;
logitem->browser_type = NULL;
logitem->continent = NULL;
logitem->asn = NULL;
logitem->country = NULL;
logitem->city = NULL;
logitem->date = NULL;
logitem->errstr = NULL;
logitem->host = NULL;
logitem->method = NULL;
logitem->os = NULL;
logitem->os_type = NULL;
logitem->protocol = NULL;
logitem->qstr = NULL;
logitem->ref = NULL;
logitem->req_key = NULL;
logitem->req = NULL;
logitem->resp_size = 0LL;
logitem->serve_time = 0;
logitem->status = -1;
logitem->time = NULL;
logitem->uniq_key = 0;
logitem->uniq_first = 0;
logitem->vhost = NULL;
logitem->userid = NULL;
logitem->cache_status = NULL;
logitem->utm_path = NULL;
/* UMS */
logitem->mime_type = NULL;
logitem->tls_type = NULL;
logitem->tls_cypher = NULL;
logitem->tls_type_cypher = NULL;
logitem->tls_type_cypher_group = NULL;
logitem->tls_group = NULL;
memset (logitem->site, 0, sizeof (logitem->site));
logitem->dt = glog->start_time;
return logitem;
}
/* Free all members of a GLogItem */
void
free_glog (GLogItem *logitem) {
if (!logitem)
return;
if (logitem->agent != NULL)
free (logitem->agent);
if (logitem->browser != NULL)
free (logitem->browser);
if (logitem->browser_type != NULL)
free (logitem->browser_type);
if (logitem->continent != NULL)
free (logitem->continent);
if (logitem->asn != NULL)
free (logitem->asn);
if (logitem->country != NULL)
free (logitem->country);
if (logitem->city != NULL)
free (logitem->city);
if (logitem->date != NULL)
free (logitem->date);
if (logitem->errstr != NULL)
free (logitem->errstr);
if (logitem->host != NULL)
free (logitem->host);
if (logitem->method != NULL)
free (logitem->method);
if (logitem->os != NULL)
free (logitem->os);
if (logitem->os_type != NULL)
free (logitem->os_type);
if (logitem->protocol != NULL)
free (logitem->protocol);
if (logitem->qstr != NULL)
free (logitem->qstr);
if (logitem->ref != NULL)
free (logitem->ref);
if (logitem->req_key != NULL)
free (logitem->req_key);
if (logitem->req != NULL)
free (logitem->req);
if (logitem->time != NULL)
free (logitem->time);
if (logitem->userid != NULL)
free (logitem->userid);
if (logitem->cache_status != NULL)
free (logitem->cache_status);
if (logitem->vhost != NULL)
free (logitem->vhost);
if (logitem->utm_path != NULL)
free (logitem->utm_path);
free_utm (&logitem->utm);
if (logitem->mime_type != NULL)
free (logitem->mime_type);
if (logitem->tls_type != NULL)
free (logitem->tls_type);
if (logitem->tls_cypher != NULL)
free (logitem->tls_cypher);
if (logitem->tls_type_cypher != NULL)
free (logitem->tls_type_cypher);
if (logitem->tls_type_cypher_group != NULL)
free (logitem->tls_type_cypher_group);
if (logitem->tls_group != NULL)
free (logitem->tls_group);
free (logitem);
}
/* Entry point to decode the given URL-encoded string.
*
* On success, the decoded trimmed string is assigned to the output
* buffer. */
static char *
decode_url (char *url) {
char *out, *decoded;
if ((url == NULL) || (*url == '\0'))
return NULL;
out = decoded = xstrdup (url);
decode_hex (url, out, 0);
/* double encoded URL? */
if (conf.double_decode)
decode_hex (decoded, out, 0);
strip_newlines (out);
return trim_str (out);
}
/* Parse a URI and extracts the *host* part from it
* i.e., //www.example.com/path?googleguy > www.example.com
*
* On error, 1 is returned.
* On success, the extracted referer is set and 0 is returned. */
static int
extract_referer_site (const char *referer, char *host) {
char *url, *begin, *end;
int len = 0;
if ((referer == NULL) || (*referer == '\0'))
return 1;
url = strdup (referer);
if ((begin = strstr (url, "//")) == NULL)
goto clean;
begin += 2;
if ((len = strlen (begin)) == 0)
goto clean;
if ((end = strpbrk (begin, "/?")) != NULL)
len = end - begin;
if (len == 0)
goto clean;
if (len >= REF_SITE_LEN)
len = REF_SITE_LEN;
memcpy (host, begin, len);
host[len] = '\0';
free (url);
return 0;
clean:
free (url);
return 1;
}
/* Determine if the given request is static (e.g., jpg, css, js, etc).
*
* On error, or if not static, 0 is returned.
* On success, the 1 is returned. */
static int
verify_static_content (const char *req) {
const char *nul = NULL;
const char *ext = NULL, *pch = NULL;
int elen = 0, i;
if ((req == NULL) || (*req == '\0'))
return 0;
nul = req + strlen (req);
for (i = 0; i < conf.static_file_idx; ++i) {
ext = conf.static_files[i];
if (ext == NULL || *ext == '\0')
continue;
elen = strlen (ext);
if (conf.all_static_files && (pch = strchr (req, '?')) != NULL && pch - req > elen) {
pch -= elen;
if (0 == strncasecmp (ext, pch, elen))
return 1;
continue;
}
if (nul - req > elen && !strncasecmp (nul - elen, ext, elen))
return 1;
}
return 0;
}
/* Extract the HTTP method.
*
* On error, or if not found, NULL is returned.
* On success, the HTTP method is returned. */
static const char *
extract_method (const char *token) {
size_t i;
for (i = 0; i < http_methods_len; i++) {
if (strncasecmp (token, http_methods[i].method, http_methods[i].len) == 0) {
return http_methods[i].method;
}
}
return NULL;
}
/* Normalize a cache status, taking ownership of the allocated token.
*
* On success, an allocated status is returned, preserving existing short values.
* On failure, the unrecognized token is freed and NULL is returned. */
static char *
normalize_cache_status (char *tkn) {
static const char *const statuses[] = {
"MISS", "BYPASS", "EXPIRED", "STALE", "UPDATING", "REVALIDATED", "HIT", "INVALIDATED"
};
/* Match mod_cache's categories, including misses logged as bare reasons.
* Prefixes include separators so unrelated words cannot match a status. */
static const GCacheStatusRule apache_statuses[] = {
{"cache hit", "HIT", 0},
{"cache hit: ", "HIT", 1},
{"conditional cache hit: ", "REVALIDATED", 1},
{"cache miss: ", "MISS", 1},
{"conditional cache miss: ", "MISS", 1},
{"cache invalidated by ", "INVALIDATED", 1},
{"Response status ", "MISS", 1},
{"Broken expires header: ", "MISS", 1},
{"Expires header already expired; not cacheable", "MISS", 0},
{"s-maxage or max-age zero and no Last-Modified or Etag; not cacheable", "MISS", 0},
{"Query string present but no explicit expiration time", "MISS", 0},
{"HTTP Status 304 Not Modified", "MISS", 0},
{"No Last-Modified; Etag; Expires; Cache-Control:max-age or Cache-Control:s-maxage headers", "MISS", 0},
{"Cache-Control: no-store present", "MISS", 0},
{"Cache-Control: private present", "MISS", 0},
{"Authorization required", "MISS", 0},
{"Vary header contains '*'", "MISS", 0},
{"environment variable 'no-cache' is set", "MISS", 0},
{"r->no_cache present", "MISS", 0},
{"updated entity is older than cached entity", "MISS", 0},
{"invalid content length", "MISS", 0},
};
const GCacheStatusRule *rule = NULL;
size_t i;
for (i = 0; i < ARRAY_SIZE (statuses); ++i) {
if (strcasecmp (statuses[i], tkn) == 0)
return tkn;
}
for (i = 0; i < ARRAY_SIZE (apache_statuses); ++i) {
rule = &apache_statuses[i];
if (rule->prefix ? strncasecmp (rule->value, tkn, strlen (rule->value)) : strcasecmp (rule->value, tkn))
continue;
free (tkn);
return xstrdup (rule->status);
}
free (tkn);
return NULL;
}
/* Determine if the given token is a valid HTTP protocol.
*
* If not valid, 1 is returned.
* If valid, 0 is returned. */
static const char *
extract_protocol (const char *token) {
size_t i;
for (i = 0; i < http_protocols_len; i++) {
if (strncasecmp (token, http_protocols[i].protocol, http_protocols[i].len) == 0) {
return http_protocols[i].protocol;
}
}
return NULL;
}
/* Parse a request containing the method and protocol.
*
* On error, or unable to parse, NULL is returned.
* On success, the HTTP request is returned and the method and
* protocol are assigned to the corresponding buffers. */
static char *
parse_req (char *line, char **method, char **protocol) {
char *req = NULL, *request = NULL, *dreq = NULL, *ptr = NULL;
const char *meth, *proto;
ptrdiff_t rlen;
meth = extract_method (line);
/* couldn't find a method, so use the whole request line */
if (meth == NULL) {
request = xstrdup (line);
}
/* method found, attempt to parse request */
else {
req = line + strlen (meth);
if (!(ptr = strrchr (req, ' ')) || !(proto = extract_protocol (++ptr)))
return alloc_string ("-");
req++;
if ((rlen = ptr - req) <= 0)
return alloc_string ("-");
request = xmalloc (rlen + 1);
strncpy (request, req, rlen);
request[rlen] = 0;
if (conf.append_method)
(*method) = strtoupper (xstrdup (meth));
if (conf.append_protocol)
(*protocol) = strtoupper (xstrdup (proto));
}
if (!(dreq = decode_url (request)))
return request;
else if (*dreq == '\0') {
free (dreq);
return request;
}
free (request);
return dreq;
}
/* Map a numeric TLS wire-protocol version to its display name.
*
* On success, a pointer to the static display name is returned.
* On failure, NULL is returned. */
static const char *
get_numeric_tls_version (const char *token) {
char *end = NULL;
unsigned long version = 0;
errno = 0;
version = strtoul (token, &end, 10);
if (token == end || *end != '\0' || errno == ERANGE)
return NULL;
switch (version) {
case TLS_VERSION_SSL3:
return "SSLv3";
case TLS_VERSION_1_0:
return "TLSv1";
case TLS_VERSION_1_1:
return "TLSv1.1";
case TLS_VERSION_1_2:
return "TLSv1.2";
case TLS_VERSION_1_3:
return "TLSv1.3";
default:
return NULL;
}
}
#if defined(HAVE_LIBSSL) && defined(HAVE_CIPHER_STD_NAME)
/* Resolve a numeric TLS cipher and consume the supplied token.
*
* On success, the resolved cipher is assigned and 0 is returned.
* On failure, 1 is returned. */
static int
extract_tls_version_cipher (char *tkn, char **cipher, char **tls_version) {
SSL_CTX *ctx = NULL;
SSL *ssl = NULL;
int code = 0;
unsigned short code_be;
unsigned char cipherid[3];
const SSL_CIPHER *c = NULL;
char *bEnd;
const char *sn = NULL;
code = strtoull (tkn, &bEnd, 10);
if (tkn == bEnd || *bEnd != '\0' || errno == ERANGE) {
LOG_DEBUG (("unable to convert cipher code to a valid decimal."));
goto fail;
}
/* ssl context */
if (!(ctx = SSL_CTX_new (SSLv23_server_method ()))) {
LOG_DEBUG (("Unable to create a new SSL_CTX_new to extract TLS."));
goto fail;
}
if (!(ssl = SSL_new (ctx))) {
LOG_DEBUG (("Unable to create a new instance of SSL_new to extract TLS."));
goto fail;
}
code_be = htobe16 (code);
memcpy (cipherid, &code_be, 2);
cipherid[2] = 0;
if (!(c = SSL_CIPHER_find (ssl, cipherid))) {
LOG_DEBUG (("Unable to find cipher to extract TLS."));
goto fail;
}
if (!(sn = SSL_CIPHER_standard_name (c))) {
LOG_DEBUG (("Unable to get cipher standard name to extract TLS."));
goto fail;
}
*cipher = xstrdup (sn);
if (!*tls_version)
*tls_version = xstrdup (SSL_CIPHER_get_version (c));
free (tkn);
SSL_free (ssl);
SSL_CTX_free (ctx);
return 0;
fail:
free (tkn);
if (ssl)
SSL_free (ssl);
if (ctx)
SSL_CTX_free (ctx);
return 1;
}
#endif
/* Extract the next delimiter given a log format and copy the delimiter to the
* destination buffer.
*
* On error, the dest buffer will be empty.
* On success, the delimiter(s) are stored in the dest buffer. */
static void
get_delim (char *dest, const char *p) {
/* done, nothing to do */
if (p[0] == '\0' || p[1] == '\0') {
dest[0] = '\0';
return;
}
/* add the first delim */
dest[0] = *(p + 1);
}
/* Extract and malloc a token given the parsed rule.
*
* On success, the malloc'd token is returned. */
static char *
parsed_string (const char *pch, const char **str, int move_ptr) {
char *p;
size_t len = (pch - *str + 1);
p = xmalloc (len);
memcpy (p, *str, (len - 1));
p[len - 1] = '\0';
if (move_ptr)
*str += len - 1;
return trim_str (p);
}
/* Find and extract a token given a log format rule.
*
* On error, or unable to parse it, NULL is returned.
* On success, the malloc'd token is returned. */
static char *
parse_string (const char **str, const char *delims, int cnt) {
int idx = 0;
const char *pch = *str, *p = NULL;
char end;
if ((*delims != 0x0) && (p = strpbrk (*str, delims)) == NULL)
return NULL;
end = !*delims ? 0x0 : *p;
do {
/* match number of delims */
if (*pch == end)
idx++;
/* delim found, parse string then */
if ((*pch == end && cnt == idx) || *pch == '\0')
return parsed_string (pch, str, 1);
/* advance to the first unescaped delim */
if (*pch == '\\')
pch++;
} while (*pch++);
return NULL;
}
/* Parse a virtual host without crossing its whitespace-delimited field.
*
* On success, the allocated virtual host is returned.
* On failure, NULL is returned. */
static char *
parse_vhost (const char **str, const char *end) {
const char *field_end = NULL, *separator = NULL;
if (*end != ':')
return parse_string (str, end, 1);
field_end = strpbrk (*str, " \t");
if (field_end == NULL)
field_end = *str + strlen (*str);
if (**str == '[') {
separator = memchr (*str, ']', field_end - *str);
if (separator == NULL || separator + 1 >= field_end || separator[1] != ':')
return NULL;
separator++;
} else {
separator = memchr (*str, ':', field_end - *str);
if (separator == NULL || separator == *str)
return NULL;
}
return parsed_string (separator, str, 1);
}
char *
extract_by_delim (const char **str, const char *end) {
return parse_string (&(*str), end, 1);
}
/* Move forward through the log string until a non-space (!isspace)
* char is found. */
static void
find_alpha (const char **str) {
const char *s = *str;
while (*s) {
if (isspace ((unsigned char) *s))
s++;
else
break;
}
*str += s - *str;
}
/* Move forward through the log string until a non-space (!isspace)
* char is found and returns the count. */
static int
find_alpha_count (const char *str) {
int cnt = 0;
const char *s = str;
while (*s) {
if (isspace ((unsigned char) *s))
s++, cnt++;
else
break;
}
return cnt;
}
/* Format the broken-down time tm to a numeric date format.
*
* On error, or unable to format the given tm, 1 is returned.
* On success, a malloc'd format is returned. */
#pragma GCC diagnostic ignored "-Wformat-nonliteral"
static int
set_date (char **fdate, struct tm tm) {
char buf[DATE_LEN] = ""; /* Ymd */
memset (buf, 0, sizeof (buf));
if (strftime (buf, DATE_LEN, conf.date_num_format, &tm) <= 0)
return 1;
*fdate = xstrdup (buf);
return 0;
}
/* Format the broken-down time tm to a numeric time format.
*
* On error, or unable to format the given tm, 1 is returned.
* On success, a malloc'd format is returned. */
static int
set_time (char **ftime, struct tm tm) {
char buf[TIME_LEN] = "";
memset (buf, 0, sizeof (buf));
if (strftime (buf, TIME_LEN, "%H:%M:%S", &tm) <= 0)
return 1;
*ftime = xstrdup (buf);
return 0;
}
/* Determine the parsing specifier error and construct a message out
* of it.
*
* On success, a malloc'd error message is assigned to the log
* structure and 1 is returned. */
static int
spec_err (GLogItem *logitem, int code, const char spec, const char *tkn) {
char *err = NULL;
const char *fmt = NULL;
switch (code) {
case ERR_SPEC_TOKN_NUL:
fmt = "Token for '%%%c' specifier is NULL.";
err = xmalloc (snprintf (NULL, 0, fmt, spec) + 1);
sprintf (err, fmt, spec);
break;
case ERR_SPEC_TOKN_INV:
fmt = "Token '%s' doesn't match specifier '%%%c'";
err = xmalloc (snprintf (NULL, 0, fmt, (tkn ? tkn : "-"), spec) + 1);
sprintf (err, fmt, (tkn ? tkn : "-"), spec);
break;
case ERR_SPEC_SFMT_MIS:
fmt = "Missing braces '%s' and ignore chars for specifier '%%%c'";
err = xmalloc (snprintf (NULL, 0, fmt, (tkn ? tkn : "-"), spec) + 1);
sprintf (err, fmt, (tkn ? tkn : "-"), spec);
break;
case ERR_SPEC_LINE_INV:
fmt = "Incompatible format due to early parsed line ending '\\0'.";
err = xmalloc (snprintf (NULL, 0, fmt, (tkn ? tkn : "-")) + 1);
sprintf (err, fmt, (tkn ? tkn : "-"));
break;
case ERR_SPEC_VHOST_PORT:
fmt = "Virtual host for '%%%c' is missing the required colon-delimited port.";
err = xmalloc (snprintf (NULL, 0, fmt, spec) + 1);
sprintf (err, fmt, spec);
break;
}
logitem->errstr = err;
return code;
}
static void
set_tm_dt_logitem (GLogItem *logitem, struct tm tm) {
logitem->dt.tm_year = tm.tm_year;
logitem->dt.tm_mon = tm.tm_mon;
logitem->dt.tm_mday = tm.tm_mday;
}
static void
set_tm_tm_logitem (GLogItem *logitem, struct tm tm) {
logitem->dt.tm_hour = tm.tm_hour;
logitem->dt.tm_min = tm.tm_min;
logitem->dt.tm_sec = tm.tm_sec;
}
static void
set_numeric_date (uint32_t *numdate, const char *date) {
int res = 0;
if ((res = str2int (date)) == -1)
FATAL ("Unable to parse date to integer %s", date);
*numdate = res;
}
static void
set_agent_hash (GLogItem *logitem) {
logitem->agent_hash = djb2 ((unsigned char *) logitem->agent);
}
static int
handle_default_case_token (const char **str, const char *p) {
const char *pch = NULL;
if ((pch = strchr (*str, p[1])) != NULL)
*str += pch - *str;
return 0;
}
static void
normalize_mime_type (const char *mime, char *out, size_t out_size) {
size_t mime_len = 0, remaining = 0;
char *temp = NULL, *dest = NULL, *ptr = NULL, *token = NULL, *end = NULL, *sep = NULL, *c = NULL;
const char *delims = NULL;
int first = 0, n = 0;
if (!mime || !out || out_size == 0)
return;
mime_len = strlen (mime);
temp = xmalloc (mime_len + 1);
memcpy (temp, mime, mime_len + 1);
out[0] = '\0';
dest = out;
remaining = out_size;
/* Set the delimiters; both ';' and ',' are significant separators */
delims = ";,";
first = 1;
ptr = temp;
/* Process each token */
while (*ptr != '\0') {
/* Skip leading whitespace */
while (*ptr && isspace ((unsigned char) *ptr))
ptr++;
if (*ptr == '\0')
break;
token = ptr;
/* Find next delimiter */
sep = strpbrk (ptr, delims);
if (sep != NULL) {
*sep = '\0';
ptr = sep + 1;
} else {
ptr += strlen (ptr);
}
/* Trim trailing whitespace on token */
end = token + strlen (token) - 1;
while (end >= token && isspace ((unsigned char) *end)) {
*end = '\0';
end--;
}
if (*token == '\0')
continue;
/* Convert token to lower-case in place */
for (c = token; *c; c++)
*c = (char) tolower ((unsigned char) *c);
/* Append separator if this is not the first token */
if (!first) {
n = snprintf (dest, remaining, "; ");
if (n < 0 || (size_t) n >= remaining) {
dest[remaining - 1] = '\0';
free (temp);
return;
}
dest += n;
remaining -= n;
} else {
first = 0;
}
/* Append the normalized token safely */
n = snprintf (dest, remaining, "%s", token);
if (n < 0 || (size_t) n >= remaining) {
dest[remaining - 1] = '\0';
free (temp);
return;
}
dest += n;
remaining -= n;
}
free (temp);
}
#pragma GCC diagnostic warning "-Wformat-nonliteral"
/* Parse the log string given log format rule.
*
* On error, or unable to parse it, 1 is returned.
* On success, the malloc'd token is assigned to a GLogItem member. */
static int
parse_specifier (GLogItem *logitem, const char **str, const char *p, const char *end) {
struct tm tm;
const char *dfmt = conf.date_format, *tfmt = conf.time_format;
const char *pch = NULL, *tls_version = NULL;
char norm_mime[MAX_MIME_OUT] = { 0 };
char *sEnd = NULL, *bEnd = NULL, *tkn = NULL;
double serve_secs = 0.0;
uint64_t bandw = 0, port = 0, serve_time = 0;
int dspc = 0, fmtspcs = 0;
errno = 0;
memset (&tm, 0, sizeof (tm));
tm.tm_isdst = -1;
tm = logitem->dt;
switch (*p) {
/* date */
case 'd':
if (logitem->date)
return handle_default_case_token (str, p);
/* Attempt to parse date format containing spaces,
* i.e., syslog date format (Jul\s15, Nov\s\s2).
* Note that it's possible a date could contain some padding, e.g.,
* Dec\s\s2 vs Nov\s22, so we attempt to take that into consideration by looking
* ahead the log string and counting the # of spaces until we find an alphanum char. */
if ((fmtspcs = count_matches (dfmt, ' ')) && (pch = strchr (*str, ' ')))
dspc = find_alpha_count (pch);
if (!(tkn = parse_string (&(*str), end, MAX (dspc, fmtspcs) + 1)))
return spec_err (logitem, ERR_SPEC_TOKN_NUL, *p, NULL);
if (str_to_time (tkn, dfmt, &tm, 1) != 0 || set_date (&logitem->date, tm) != 0) {
spec_err (logitem, ERR_SPEC_TOKN_INV, *p, tkn);
free (tkn);
return 1;
}
set_numeric_date (&logitem->numdate, logitem->date);
set_tm_dt_logitem (logitem, tm);
free (tkn);
break;
/* time */
case 't':
if (logitem->time)
return handle_default_case_token (str, p);
if (!(tkn = parse_string (&(*str), end, 1)))
return spec_err (logitem, ERR_SPEC_TOKN_NUL, *p, NULL);
if (str_to_time (tkn, tfmt, &tm, 1) != 0 || set_time (&logitem->time, tm) != 0) {
spec_err (logitem, ERR_SPEC_TOKN_INV, *p, tkn);
free (tkn);
return 1;
}
set_tm_tm_logitem (logitem, tm);
free (tkn);
break;
/* date/time as decimal, i.e., timestamps, ms/us */
case 'x':
if (logitem->time && logitem->date)
return handle_default_case_token (str, p);
if (!(tkn = parse_string (&(*str), end, 1)))
return spec_err (logitem, ERR_SPEC_TOKN_NUL, *p, NULL);
if (str_to_time (tkn, tfmt, &tm, 1) != 0 || set_date (&logitem->date, tm) != 0 ||
set_time (&logitem->time, tm) != 0) {
spec_err (logitem, ERR_SPEC_TOKN_INV, *p, tkn);
free (tkn);
return 1;
}
set_numeric_date (&logitem->numdate, logitem->date);
set_tm_dt_logitem (logitem, tm);
set_tm_tm_logitem (logitem, tm);
free (tkn);
break;
/* Virtual Host */
case 'v':
if (logitem->vhost)
return handle_default_case_token (str, p);
tkn = parse_vhost (&(*str), end);
if (tkn == NULL && *end == ':')
return spec_err (logitem, ERR_SPEC_VHOST_PORT, *p, NULL);
if (tkn == NULL)
return spec_err (logitem, ERR_SPEC_TOKN_NUL, *p, NULL);
logitem->vhost = tkn;
break;
case 'p':
if (!(tkn = parse_string (&(*str), end, 1)))
return spec_err (logitem, ERR_SPEC_TOKN_NUL, *p, NULL);
for (bEnd = tkn; *bEnd != '\0'; bEnd++) {
if (!isdigit ((unsigned char) *bEnd)) {
spec_err (logitem, ERR_SPEC_TOKN_INV, *p, tkn);
free (tkn);
return 1;
}
}
errno = 0;
port = strtoull (tkn, &bEnd, 10);
if (tkn == bEnd || *bEnd != '\0' || errno == ERANGE || port > UINT16_MAX) {
spec_err (logitem, ERR_SPEC_TOKN_INV, *p, tkn);
free (tkn);
return 1;
}
free (tkn);
break;
/* remote user */
case 'e':
if (logitem->userid)
return handle_default_case_token (str, p);
tkn = parse_string (&(*str), end, 1);
if (tkn == NULL)
return spec_err (logitem, ERR_SPEC_TOKN_NUL, *p, NULL);
logitem->userid = tkn;
break;
/* cache status */
case 'C':
if (logitem->cache_status)
return handle_default_case_token (str, p);
tkn = parse_string (&(*str), end, 1);
if (tkn == NULL)
return spec_err (logitem, ERR_SPEC_TOKN_NUL, *p, NULL);
logitem->cache_status = normalize_cache_status (tkn);
break;
/* remote hostname (IP only) */
case 'h':
if (logitem->host)
return handle_default_case_token (str, p);
/* per https://datatracker.ietf.org/doc/html/rfc3986#section-3.2.2 */
/* square brackets are possible */
if (*str[0] == '[' && (*str += 1) && **str)
end = "]";
if (!(tkn = parse_string (&(*str), end, 1)))
return spec_err (logitem, ERR_SPEC_TOKN_NUL, *p, NULL);
if (!conf.no_ip_validation && invalid_ipaddr (tkn, &logitem->type_ip)) {
spec_err (logitem, ERR_SPEC_TOKN_INV, *p, tkn);
free (tkn);
return 1;
}
/* require a valid host token (e.g., ord38s18-in-f14.1e100.net) even when we're
* not validating the IP */
if (conf.no_ip_validation && *tkn == '\0') {
spec_err (logitem, ERR_SPEC_TOKN_INV, *p, tkn);
free (tkn);
return 1;
}
logitem->host = tkn;
break;
/* request method */
case 'm':
if (logitem->method)
return handle_default_case_token (str, p);
if (!(tkn = parse_string (&(*str), end, 1)))
return spec_err (logitem, ERR_SPEC_TOKN_NUL, *p, NULL);
{
const char *meth = NULL;
if (!(meth = extract_method (tkn))) {
spec_err (logitem, ERR_SPEC_TOKN_INV, *p, tkn);
free (tkn);
return 1;
}
logitem->method = xstrdup (meth);
free (tkn);
}
break;
/* request not including method or protocol */
case 'U':
if (logitem->req)
return handle_default_case_token (str, p);
tkn = parse_string (&(*str), end, 1);
if (tkn == NULL || *tkn == '\0') {
free (tkn);
return spec_err (logitem, ERR_SPEC_TOKN_NUL, *p, NULL);
}
extract_utm (tkn, UTM_INPUT_URI, &logitem->utm);
if ((logitem->req = decode_url (tkn)) == NULL) {
spec_err (logitem, ERR_SPEC_TOKN_INV, *p, tkn);
free (tkn);
return 1;
}
free (tkn);
break;
/* query string alone, e.g., ?param=goaccess&tbm=shop */
case 'q':
if (logitem->qstr)
return handle_default_case_token (str, p);
tkn = parse_string (&(*str), end, 1);
if (tkn == NULL || *tkn == '\0' || !strcmp (tkn, MISSING_LOG_VALUE)) {
free (tkn);
return 0;
}
extract_utm (tkn, UTM_INPUT_QUERY, &logitem->utm);
if ((logitem->qstr = decode_url (tkn)) == NULL) {
spec_err (logitem, ERR_SPEC_TOKN_INV, *p, tkn);
free (tkn);
return 1;
}
free (tkn);
break;
/* request protocol */
case 'H':
if (logitem->protocol)
return handle_default_case_token (str, p);
if (!(tkn = parse_string (&(*str), end, 1)))
return spec_err (logitem, ERR_SPEC_TOKN_NUL, *p, NULL);
{
const char *proto = NULL;
if (!(proto = extract_protocol (tkn))) {
spec_err (logitem, ERR_SPEC_TOKN_INV, *p, tkn);
free (tkn);
return 1;
}
logitem->protocol = xstrdup (proto);
free (tkn);
}
break;
/* request, including method + protocol */
case 'r':
if (logitem->req)
return handle_default_case_token (str, p);
if (!(tkn = parse_string (&(*str), end, 1)))
return spec_err (logitem, ERR_SPEC_TOKN_NUL, *p, NULL);
extract_utm (tkn, UTM_INPUT_URI, &logitem->utm);
logitem->req = parse_req (tkn, &logitem->method, &logitem->protocol);
free (tkn);
break;
/* Status Code */
case 's':
if (logitem->status >= 0)
return handle_default_case_token (str, p);
if (!(tkn = parse_string (&(*str), end, 1)))
return spec_err (logitem, ERR_SPEC_TOKN_NUL, *p, NULL);
logitem->status = strtol (tkn, &sEnd, 10);
if (tkn == sEnd || *sEnd != '\0' || errno == ERANGE ||
(!conf.no_strict_status && !is_valid_http_status (logitem->status))) {
spec_err (logitem, ERR_SPEC_TOKN_INV, *p, tkn);
free (tkn);
return 1;
}
free (tkn);
break;
/* size of response in bytes - excluding HTTP headers */
case 'b':
if (logitem->resp_size)
return handle_default_case_token (str, p);
if (!(tkn = parse_string (&(*str), end, 1)))
return spec_err (logitem, ERR_SPEC_TOKN_NUL, *p, NULL);
bandw = strtoull (tkn, &bEnd, 10);
if (tkn == bEnd || *bEnd != '\0' || errno == ERANGE)
bandw = 0;
logitem->resp_size = bandw;
{
int expected = 0;
__atomic_compare_exchange_n (&conf.bandwidth, &expected, 1, false, __ATOMIC_SEQ_CST,
__ATOMIC_SEQ_CST);
}
free (tkn);
break;
/* referrer */
case 'R':
if (logitem->ref)
return handle_default_case_token (str, p);
if (!(tkn = parse_string (&(*str), end, 1)))
tkn = alloc_string ("-");
if (*tkn == '\0') {
free (tkn);
tkn = alloc_string ("-");
}
if (strcmp (tkn, "-") != 0) {
extract_referer_site (tkn, logitem->site);
/* hide referrers from report */
if (hide_referer (logitem->site)) {
logitem->site[0] = '\0';
free (tkn);
} else
logitem->ref = tkn;
break;
}
logitem->ref = tkn;
break;
/* user agent */
case 'u':
if (logitem->agent)
return handle_default_case_token (str, p);
tkn = parse_string (&(*str), end, 1);
if (tkn != NULL && *tkn != '\0') {
/* Make sure the user agent is decoded (i.e.: CloudFront) */
logitem->agent = decode_url (tkn);
set_browser_os (logitem);
set_agent_hash (logitem);
free (tkn);
break;
} else if (tkn != NULL && *tkn == '\0') {
free (tkn);
tkn = alloc_string ("-");
}
/* must be null */
else {
tkn = alloc_string ("-");
}
logitem->agent = tkn;
set_agent_hash (logitem);
break;
/* time taken to serve the request, in milliseconds as a decimal number */
case 'L':
/* ignore it if we already have served time */
if (logitem->serve_time)
return handle_default_case_token (str, p);
if (!(tkn = parse_string (&(*str), end, 1)))
return spec_err (logitem, ERR_SPEC_TOKN_NUL, *p, NULL);
serve_secs = strtoull (tkn, &bEnd, 10);
if (tkn == bEnd || *bEnd != '\0' || errno == ERANGE)
serve_secs = 0;
/* convert it to microseconds */
logitem->serve_time = (serve_secs > 0) ? serve_secs * MILS : 0;
/* Determine if time-served data was stored on-disk. */
{
int expected = 0;
__atomic_compare_exchange_n (&conf.serve_usecs, &expected, 1, false, __ATOMIC_SEQ_CST,
__ATOMIC_SEQ_CST);
}
free (tkn);
break;
/* time taken to serve the request, in seconds with a milliseconds
* resolution */
case 'T':
/* ignore it if we already have served time */
if (logitem->serve_time)
return handle_default_case_token (str, p);
if (!(tkn = parse_string (&(*str), end, 1)))
return spec_err (logitem, ERR_SPEC_TOKN_NUL, *p, NULL);
if (strchr (tkn, '.') != NULL)
serve_secs = strtod (tkn, &bEnd);
else
serve_secs = strtoull (tkn, &bEnd, 10);
if (tkn == bEnd || *bEnd != '\0' || errno == ERANGE)
serve_secs = 0;
/* convert it to microseconds */
logitem->serve_time = (serve_secs > 0) ? serve_secs * SECS : 0;
/* Determine if time-served data was stored on-disk. */
{
int expected = 0;
__atomic_compare_exchange_n (&conf.serve_usecs, &expected, 1, false, __ATOMIC_SEQ_CST,
__ATOMIC_SEQ_CST);
}
free (tkn);
break;
/* time taken to serve the request, in microseconds */
case 'D':
/* ignore it if we already have served time */
if (logitem->serve_time)
return handle_default_case_token (str, p);
if (!(tkn = parse_string (&(*str), end, 1)))
return spec_err (logitem, ERR_SPEC_TOKN_NUL, *p, NULL);
serve_time = strtoull (tkn, &bEnd, 10);
if (tkn == bEnd || *bEnd != '\0' || errno == ERANGE)
serve_time = 0;
logitem->serve_time = serve_time;
/* Determine if time-served data was stored on-disk. */
{
int expected = 0;
__atomic_compare_exchange_n (&conf.serve_usecs, &expected, 1, false, __ATOMIC_SEQ_CST,
__ATOMIC_SEQ_CST);
}
free (tkn);
break;
/* time taken to serve the request, in nanoseconds */
case 'n':
/* ignore it if we already have served time */
if (logitem->serve_time)
return handle_default_case_token (str, p);
if (!(tkn = parse_string (&(*str), end, 1)))
return spec_err (logitem, ERR_SPEC_TOKN_NUL, *p, NULL);
serve_time = strtoull (tkn, &bEnd, 10);
if (tkn == bEnd || *bEnd != '\0' || errno == ERANGE)
serve_time = 0;
/* convert it to microseconds */
logitem->serve_time = (serve_time > 0) ? serve_time / MILS : 0;
/* Determine if time-served data was stored on-disk. */
{
int expected = 0;
__atomic_compare_exchange_n (&conf.serve_usecs, &expected, 1, false, __ATOMIC_SEQ_CST,
__ATOMIC_SEQ_CST);
}
free (tkn);
break;
/* TLS key exchange group, e.g., "X25519MLKEM768". */
case 'g':
if (logitem->tls_group)
return handle_default_case_token (str, p);
tkn = parse_string (&(*str), end, 1);
if (tkn == NULL)
return spec_err (logitem, ERR_SPEC_TOKN_NUL, *p, NULL);
if (*tkn == '\0' || !strcmp (tkn, MISSING_LOG_VALUE)) {
free (tkn);
break;
}
logitem->tls_group = tkn;
break;
/* UMS: Krypto (TLS) "ECDHE-RSA-AES128-GCM-SHA256" */
case 'k':
/* error to set this twice */
if (logitem->tls_cypher)
return handle_default_case_token (str, p);
if (!(tkn = parse_string (&(*str), end, 1)))
return spec_err (logitem, ERR_SPEC_TOKN_NUL, *p, NULL);
#if defined(HAVE_LIBSSL) && defined(HAVE_CIPHER_STD_NAME)
{
char *tmp = NULL;
for (tmp = tkn; isdigit ((unsigned char) *tmp); tmp++);
if (!strlen (tmp))
extract_tls_version_cipher (tkn, &logitem->tls_cypher, &logitem->tls_type);
else
logitem->tls_cypher = tkn;
}
#else
logitem->tls_cypher = tkn;
#endif
break;
/* UMS: Krypto (TLS) parameters like "TLSv1.2" */
case 'K':
/* error to set this twice */
if (logitem->tls_type)
return handle_default_case_token (str, p);
if (!(tkn = parse_string (&(*str), end, 1)))
return spec_err (logitem, ERR_SPEC_TOKN_NUL, *p, NULL);
if ((tls_version = get_numeric_tls_version (tkn))) {
free (tkn);
tkn = xstrdup (tls_version);
}
logitem->tls_type = tkn;
break;
/* UMS: Mime-Type like "text/html" */
case 'M':
/* error to set this twice */
if (logitem->mime_type)
return handle_default_case_token (str, p);
if (!(tkn = parse_string (&(*str), end, 1)))
return spec_err (logitem, ERR_SPEC_TOKN_NUL, *p, NULL);
normalize_mime_type (tkn, norm_mime, sizeof (norm_mime));
if (norm_mime[0] != '\0')
logitem->mime_type = strdup (norm_mime);
else
logitem->mime_type = NULL;
free (tkn);
break;
/* move forward through str until not a space */
case '~':
find_alpha (&(*str));
break;
/* everything else skip it */
default:
handle_default_case_token (str, p);
}
return 0;
}
/* Parse the special host specifier and extract the characters that
* need to be rejected when attempting to parse the XFF field.
*
* If no unable to find both curly braces (boundaries), NULL is returned.
* On success, the malloc'd reject set is returned. */
static char *
extract_braces (const char **p) {
const char *b1 = NULL, *b2 = NULL, *s = *p;
char *ret = NULL;
int esc = 0;
ptrdiff_t len = 0;
/* iterate over the log format */
for (; *s; s++) {
if (*s == '\\') {
esc = 1;
} else if (*s == '{' && !esc) {
b1 = s;
} else if (*s == '}' && !esc) {
b2 = s;
break;
} else {
esc = 0;
}
}
if ((!b1) || (!b2))
return NULL;
if ((len = b2 - (b1 + 1)) <= 0)
return NULL;
/* Found braces, extract 'reject' character set. */
ret = xmalloc (len + 1);
memcpy (ret, b1 + 1, len);
ret[len] = '\0';
(*p) = b2 + 1;
return ret;
}
/* Attempt to extract the client IP from an X-Forwarded-For (XFF) field.
*
* If no IP is found, 1 is returned.
* On success, the malloc'd token is assigned to a GLogItem->host and
* 0 is returned. */
static int
set_xff_host (GLogItem *logitem, const char *str, const char *skips, int out) {
const char *ptr = NULL, *tkn = NULL;
int invalid_ip = 1, len = 0, type_ip = TYPE_IPINV;
int idx = 0, skips_len = 0;
skips_len = strlen (skips);
ptr = str;
while (*ptr != '\0') {
if ((len = strcspn (ptr, skips)) == 0) {
len++, ptr++, idx++;
goto move;
}
/* If our index does not match the number of delimiters and we have already a
* valid client IP, then we assume we have reached the length of the XFF */
if (idx < skips_len && logitem->host)
break;
ptr += len;
/* extract possible IP */
if (!(tkn = parsed_string (ptr, &str, 0)))
break;
invalid_ip = invalid_ipaddr (tkn, &type_ip);
/* done, already have IP and current token is not a host */
if (logitem->host && invalid_ip) {
free ((void *) tkn);
break;
}
if (!logitem->host && !invalid_ip) {
logitem->host = xstrdup (tkn);
logitem->type_ip = type_ip;
}
free ((void *) tkn);
idx = 0;
/* found the client IP, break then */
if (logitem->host && out)
break;
move:
str += len;
}
return logitem->host == NULL;
}
/* Attempt to find possible delimiters in the X-Forwarded-For (XFF) field.
*
* If no IP is found, 1 is returned.
* On success, the malloc'd token is assigned to a GLogItem->host and 0 is returned. */
static int
find_xff_host (GLogItem *logitem, const char **str, const char **p) {
char *skips = NULL, *extract = NULL;
char pch[2] = { 0 };
int res = 0;
if (!(skips = extract_braces (p)))
return spec_err (logitem, ERR_SPEC_SFMT_MIS, **p, "{}");
/* if the log format current char is not within the braces special chars, then
* we assume the range of IPs are within hard delimiters */
if (!strchr (skips, **p) && strchr (*str, **p)) {
*pch = **p;
*(pch + 1) = '\0';
if (!(extract = parse_string (&(*str), pch, 1)))
goto clean;
res = set_xff_host (logitem, extract, skips, 1);
free (extract);
(*str)++; /* move a char forward from the trailing delim */
} else {
res = set_xff_host (logitem, *str, skips, 0);
}
clean:
free (skips);
return res;
}
/* Handle special specifiers.
*
* On error, or unable to parse it, 1 is returned.
* On success, the malloc'd token is assigned to a GLogItem member and
* 0 is returned. */
static int
special_specifier (GLogItem *logitem, const char **str, const char **p) {
switch (**p) {
/* XFF remote hostname (IP only) */
case 'h':
if (find_xff_host (logitem, str, p))
return spec_err (logitem, ERR_SPEC_TOKN_NUL, 'h', NULL);
break;
}
return 0;
}
/* Iterate over the given log format.
*
* On error, or unable to parse it, 1 is returned.
* On success, the malloc'd token is assigned to a GLogItem member and
* 0 is returned. */
static int
parse_format (GLogItem *logitem, const char *str, const char *lfmt) {
char end[2 + 1] = { 0 };
const char *p = NULL, *last = NULL;
int perc = 0, tilde = 0, ret = 0;
if (str == NULL || *str == '\0')
return 1;
/* iterate over the log format */
last = lfmt + strlen (lfmt);
for (p = lfmt; p < last; p++) {
if (*p == '%') {
perc++;
continue;
}
if (*p == '~' && perc == 0) {
tilde++;
continue;
}
if (*str == '\0')
return spec_err (logitem, ERR_SPEC_LINE_INV, '-', NULL);
if (*str == '\n')
return 0;
if (tilde && *p != '\0') {
if (*str == '\0')
return 0;
if (special_specifier (logitem, &str, &p) == 1)
return 1;
tilde = 0;
}
/* %h */
else if (perc && *p != '\0') {
if (*str == '\0')
return 0;
memset (end, 0, sizeof end);
get_delim (end, p);
/* attempt to parse format specifiers */
if ((ret = parse_specifier (logitem, &str, p, end)))
return ret;
perc = 0;
} else if (perc && isspace ((unsigned char) p[0])) {
return 1;
} else {
str++;
}
}
return 0;
}
/* Determine if the log string is valid and if it's not a comment.
*
* On error, or invalid, 1 is returned.
* On success, or valid line, 0 is returned. */
static int
valid_line (char *line) {
/* invalid line */
if ((line == NULL) || (*line == '\0'))
return 1;
/* ignore comments */
if (*line == '#' || *line == '\n')
return 1;
return 0;
}
/* Ignore request's query string. e.g.,
* /index.php?timestamp=1454385289 */
static void
strip_qstring (char *req) {
char *qmark;
if ((qmark = strchr (req, '?')) != NULL) {
if ((qmark - req) > 0)
*qmark = '\0';
}
}
/* Output all log errors stored during parsing. */
void
output_logerrors (void) {
Logs *logs = get_db_logs (DB_INSTANCE);
GLog *glog = NULL;
int pid = getpid (), i = 0, j = 0;
uint8_t nerrors = 0;
for (i = 0; i < logs->size; ++i) {
glog = &logs->glog[i];
/* Snapshot the atomic error count once. A distinct inner index keeps
* the outer file index monotonic (a shared index would let the inner
* loop rewind it and, with more sources than errors, never terminate),
* and the snapshot keeps the printed count consistent with the lines
* emitted even if a parse thread is still appending. */
nerrors = atomic_load (&glog->log_erridx);
if (!nerrors)
continue;
fprintf (stderr, "==%d== GoAccess - version %s - %s %s\n", pid, GO_VERSION, __DATE__, __TIME__);
fprintf (stderr, "==%d== Config file: %s\n", pid, conf.iconfigfile ? : NO_CONFIG_FILE);
fprintf (stderr, "==%d== https://goaccess.io - <hello@goaccess.io>\n", pid);
fprintf (stderr, "==%d== Released under the MIT License.\n", pid);
fprintf (stderr, "==%d==\n", pid);
fprintf (stderr, "==%d== FILE: %s\n", pid, glog->props.filename);
fprintf (stderr, "==%d== ", pid);
fprintf (stderr, ERR_PARSED_NLINES, nerrors);
fprintf (stderr, " %s:\n", ERR_PARSED_NLINES_DESC);
fprintf (stderr, "==%d==\n", pid);
for (j = 0; j < nerrors; ++j)
fprintf (stderr, "==%d== %s\n", pid, glog->errors[j]);
}
fprintf (stderr, "==%d==\n", pid);
fprintf (stderr, "==%d== %s\n", pid, ERR_FORMAT_HEADER);
fprintf (stderr, "==%d== %s\n", pid, HINT_INVALID_REQUESTS);
}
/* Ensure we have the following fields. */
static int
verify_missing_fields (GLogItem *logitem) {
/* must have the following fields */
if (logitem->host == NULL)
logitem->errstr =
xstrdup
("IPv4/6 is required. You have to add format specifier '%h' [host (the client IP address, either IPv4 or IPv6)] to your log-format.");
else if (logitem->date == NULL)
logitem->errstr =
xstrdup
("A valid date is required. You have to add format specifier '%x' [Datetime] or '%d' [Date] and '%t' [Time] to your log-format.");
else if (logitem->req == NULL)
logitem->errstr =
xstrdup
("A request is required. Your log-format is missing format specifier '%r' [The request line from the client] or combination of special format specifiers such as '%m', '%U', '%q' and '%H' to parse individual fields.");
return logitem->errstr != NULL;
}
/* Determine if the request is from a robot or spider and check if we
* need to ignore or show crawlers only.
*
* On success (the request should be kept), 1 is returned.
* On failure (the request should be ignored), 0 is returned. */
static int
handle_crawler (const GLogItem *logitem) {
int bot = 0;
if (!conf.ignore_crawlers && !conf.crawlers_only)
return 1;
if (!logitem->browser_type && !logitem->os_type)
bot = is_crawler (logitem->agent);
else
bot = is_crawler_type (logitem->browser_type) || is_crawler_type (logitem->os_type);
return (conf.ignore_crawlers && bot) || (conf.crawlers_only && !bot) ? 0 : 1;
}
/* A wrapper function to determine if the request is static.
*
* If the request is not static, 0 is returned.
* If the request is static, 1 is returned. */
static int
is_static (const char *req) {
return verify_static_content (req);
}
/* Determine if the request of the given status code needs to be
* ignored.
*
* If the status code is not within the ignore-array, 0 is returned.
* If the status code is within the ignore-array, 1 is returned. */
static int
ignore_status_code (int status) {
int i = 0;
if (!status || conf.ignore_status_idx == 0)
return 0;
for (i = 0; i < conf.ignore_status_idx; i++)
if (status == conf.ignore_status[i])
return 1;
return 0;
}
/* Determine if static file request should be ignored
*
* If the request line is not ignored, 0 is returned.
* If the request line is ignored, 1 is returned. */
static int
ignore_static (const char *req) {
if (conf.ignore_statics && is_static (req))
return 1;
return 0;
}
/* Determine if the request status code is a 404.
*
* If the request is not a 404, 0 is returned.
* If the request is a 404, 1 is returned. */
static int
is_404 (GLogItem *logitem) {
/* is this a 404? */
if (logitem->status == 404)
return 1;
/* treat 444 as 404? */
else if (logitem->status == 444 && conf.code444_as_404)
return 1;
return 0;
}
/* A wrapper function to determine if a log line needs to be ignored.
*
* If the request line is not ignored, 0 is returned.
* If the request line is ignored, IGNORE_LEVEL_PANEL is returned.
* If the request line is only not counted as valid, IGNORE_LEVEL_REQ is returned. */
static int
ignore_line (GLogItem *logitem) {
if (excluded_ip (logitem) == 0)
return IGNORE_LEVEL_PANEL;
#ifdef HAVE_LIBMAXMINDDB
if (excluded_asn (logitem) == 0)
return IGNORE_LEVEL_PANEL;
#endif
if (handle_crawler (logitem) == 0)
return IGNORE_LEVEL_PANEL;
if (ignore_referer (logitem->ref))
return IGNORE_LEVEL_PANEL;
if (ignore_status_code (logitem->status))
return IGNORE_LEVEL_PANEL;
if (ignore_static (logitem->req))
return conf.ignore_statics; // IGNORE_LEVEL_PANEL or IGNORE_LEVEL_REQ
/* check if we need to remove the request's query string */
if (conf.ignore_qstr)
strip_qstring (logitem->req);
return 0;
}
/* The following generates a unique fingerprint to identify unique visitors.
* The fingerprint is made out of the IP and user agent hash; the date is
* implied by the per-date storage partitioning.
*
* On success the unique visitor fingerprint is returned */
static uint64_t
get_uniq_visitor_key (GLogItem *logitem) {
return anonymize_visitor (visitor_fingerprint (logitem->host, logitem->agent_hash));
}
/* Compare a stored prefix fingerprint with the current log.
*
* On a matching prefix, or absent prior file size, non-zero is returned.
* On a different or unavailable prefix, 0 is returned. */
static int
is_likely_same_log (GLog *glog, const GLastParse *lp) {
const uint8_t *fingerprint = NULL;
if (!lp->size)
return 1;
if (!glog->snippet[0] || !lp->snippetlen)
return 0;
if (lp->snippetlen == glog->lp.snippetlen)
fingerprint = glog->lp.fingerprint;
else if (lp->snippetlen == glog->restore_snippetlen)
fingerprint = glog->restore_fingerprint;
if (fingerprint)
return memcmp (fingerprint, lp->fingerprint, SHA256_DIGEST_SIZE) == 0;
return 0;
}
/* Determine if we should insert new record or if it's a duplicate record from
* a previously persisted dataset
*
* Returns 1 if it thinks the record it's being restored from disk
* Returns 0 if we need to parse the record */
static int
should_restore_from_disk (GLog *glog) {
GLastParse lp = { 0 };
if (!conf.restore)
return 0;
lp = ht_get_last_parse (glog->props.inode);
/* No last parse timestamp, continue parsing as we got nothing to compare
* against */
if (!lp.ts)
return 0;
/* If our current line is greater or equal (zero indexed) to the last parsed
* line and have equal timestamps, then keep parsing then */
if (glog->props.inode && is_likely_same_log (glog, &lp)) {
if (glog->props.size > lp.size && glog->read >= lp.line)
return 0;
return 1;
}
/* No inode (probably a pipe), prior or equal timestamps means restore from
* disk (exclusive) */
if (!glog->props.inode && lp.ts >= glog->lp.ts)
return 1;
/* If not likely the same content, then fallback to the following checks */
/* If timestamp is greater than last parsed, read the line then */
if (glog->lp.ts > lp.ts)
return 0;
/* Check if current log size is smaller than the one last parsed, if it is,
* it was possibly truncated and thus it may be smaller, so fallback to
* timestamp even if they are equal to the last parsed timestamp */
else if (glog->props.size < lp.size && glog->lp.ts == lp.ts)
return 0;
/* Everything else we ignore it. For instance, if current log size is
* greater than the one last parsed, or the timestamp are equal, we ignore the
* request.
*
* **NOTE* We try to play safe here as we would rather miss a few lines
* than double-count a few. */
return 1;
}
static void
process_invalid (GLog *glog, GLogItem *logitem, const char *line) {
GLastParse lp = { 0 };
/* if not restoring from disk, then count entry as proceeded and invalid */
if (!conf.restore) {
count_process_and_invalid (glog, logitem, line);
return;
}
lp = ht_get_last_parse (glog->props.inode);
/* If our current line is greater or equal (zero indexed) to the last parsed
* line then keep parsing then */
if (glog->props.inode && is_likely_same_log (glog, &lp)) {
/* only count invalids if we're past the last parsed line */
if (glog->props.size > lp.size && glog->read >= lp.line)
count_process_and_invalid (glog, logitem, line);
return;
}
/* no timestamp to compare against, just count the invalid then */
if (!logitem->numdate) {
count_process_and_invalid (glog, logitem, line);
return;
}
/* if there's a valid timestamp, count only if greater than last parsed ts */
if ((glog->lp.ts = mktime (&logitem->dt)) == -1)
return;
/* check if we were able to at least parse the date/time, if no date/time
* then we simply don't count the entry as proceed & invalid to attempt over
* counting restored data */
if (should_restore_from_disk (glog) == 0)
count_process_and_invalid (glog, logitem, line);
}
static int
parse_json_specifier (void *ptr_data, char *key, char *str) {
GLogItem *logitem = (GLogItem *) ptr_data;
char *spec = NULL;
int ret = 0;
if (!key || !str)
return 0;
/* empty JSON value, e.g., {method: ""} */
if (0 == strlen (str))
return 0;
if (!(spec = ht_get_json_logfmt (key)))
return 0;
ret = parse_format (logitem, str, spec);
free (spec);
return ret;
}
static int
parse_json_format (GLogItem *logitem, char *str) {
return parse_json_string (logitem, str, parse_json_specifier);
}
/* Atomically updates glog->lp.ts with the maximum timestamp value from
* logitem->dt.
*
* On error (if mktime fails), returns -1.
* On success, returns the updated timestamp value, which is also stored in
* glog->lp.ts.
*/
static int
atomic_lpts_update (GLog *glog, GLogItem *logitem) {
int64_t newts = mktime (&logitem->dt); // Get timestamp from logitem->dt
int64_t oldts = __atomic_load_n (&glog->lp.ts, __ATOMIC_SEQ_CST);
int64_t expected;
while (oldts < newts) { // Only update if new timestamp is later
expected = oldts;
// Attempt to update glog->lp.ts from expected (oldts) to newts.
if (__atomic_compare_exchange_n (&glog->lp.ts, &expected, newts,
false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)) {
break; // Successful update.
}
// If the CAS failed, expected now holds the current glog->lp.ts.
oldts = expected;
if (oldts >= newts)
break;
}
return newts;
}
static int
cleanup_logitem (int ret, GLogItem *logitem) {
if (logitem)
free_glog (logitem);
return ret;
}
/* Validate the basic log line format and parse it into a logitem.
*
* On error, ret is set and logitem contains error info.
* On success, 0 is returned. */
static int
validate_and_parse_line (char *line, GLogItem *logitem) {
char *fmt = conf.log_format;
int ret = 0;
/* Parse a line of log, and fill structure with appropriate values */
if (conf.is_json_log_format)
ret = parse_json_format (logitem, line);
else
ret = parse_format (logitem, line, fmt);
return ret;
}
/* Collect error messages without incrementing counters (for dry run/testing) */
static void
collect_invalid_errors (GLog *glog, GLogItem *logitem) {
uint8_t idx = 0;
if (logitem->errstr) {
pthread_mutex_lock (&glog->error_mutex);
idx = atomic_load (&glog->log_erridx);
if (idx < MAX_LOG_ERRORS) {
glog->errors[idx] = xstrdup (logitem->errstr);
atomic_store (&glog->log_erridx, idx + 1);
}
pthread_mutex_unlock (&glog->error_mutex);
}
}
/* Handle invalid log lines during parsing.
*
* During dry_run, only collects errors for display.
* During normal runs, also increments counters. */
static int
handle_invalid_line (GLog *glog, GLogItem *logitem, const char *line, int dry_run) {
/* During dry run, only collect errors without counting */
if (dry_run)
collect_invalid_errors (glog, logitem);
else
process_invalid (glog, logitem, line);
return cleanup_logitem (1, logitem);
}
/* Apply post-parse processing to a valid logitem.
* This includes vhost handling, timestamp updates, and restoration checks.
*
* Returns 1 if the line should be skipped (restoration check).
* Returns 0 if processing should continue. */
static int
apply_post_parse_processing (GLog *glog, GLogItem *logitem) {
/* Apply vhost from filename if configured */
if (!glog->piping && conf.fname_as_vhost && glog->fname_as_vhost)
logitem->vhost = xstrdup (glog->fname_as_vhost);
/* Update timestamp atomically */
if (atomic_lpts_update (glog, logitem) == -1)
return 1;
/* Skip if we should restore from disk */
if (should_restore_from_disk (glog))
return 1;
return 0;
}
/* Enrich a valid logitem with additional metadata.
* This includes agent handling, ignore levels, static file detection, etc. */
static void
enrich_logitem (GLogItem *logitem) {
/* agent will be null in cases where %u is not specified */
if (logitem->agent == NULL) {
logitem->agent = alloc_string ("-");
set_agent_hash (logitem);
}
logitem->ignorelevel = ignore_line (logitem);
if (is_404 (logitem))
logitem->is_404 = 1;
else if (is_static (logitem->req))
logitem->is_static = 1;
/* concatenate vhost to request */
if (conf.concat_vhost_req) {
size_t vhost_len = logitem->vhost ? strlen (logitem->vhost) : 0;
size_t req_len = logitem->req ? strlen (logitem->req) : 0;
char *new_req = xmalloc (vhost_len + req_len + 1);
if (vhost_len)
memcpy (new_req, logitem->vhost, vhost_len);
if (req_len)
memcpy (new_req + vhost_len, logitem->req, req_len);
new_req[vhost_len + req_len] = '\0';
free (logitem->req);
logitem->req = new_req;
}
logitem->uniq_key = get_uniq_visitor_key (logitem);
}
/* Parse a line from the log and store it accordingly taking into
* account multiple parsing options prior to setting data into the
* corresponding data structure.
*
* On error, logitem->errstr will contains the error message. */
int
parse_line (GLog *glog, char *line, int dry_run, GLogItem **logitem_out) {
int ret = 0;
GLogItem *logitem = NULL;
/* soft ignore these lines */
if (valid_line (line))
return -1;
logitem = init_log_item (glog);
/* Validate and parse the log format */
ret = validate_and_parse_line (line, logitem);
if (ret)
return handle_invalid_line (glog, logitem, line, dry_run);
/* Apply post-parse processing (vhost, timestamp, restoration) */
ret = apply_post_parse_processing (glog, logitem);
if (ret)
return cleanup_logitem (0, logitem);
/* valid format but missing fields */
if ((ret = verify_missing_fields (logitem)))
return handle_invalid_line (glog, logitem, line, dry_run);
/* Only count processed during non-dry runs */
if (!dry_run)
count_process (glog);
/* testing log only - return without further processing */
if (dry_run)
return cleanup_logitem (0, logitem);
/* Enrich the logitem with additional metadata */
enrich_logitem (logitem);
/* ignore line at panel level */
if (logitem->ignorelevel == IGNORE_LEVEL_PANEL)
return cleanup_logitem (0, logitem);
*logitem_out = logitem;
return 0;
}
/* Entry point to process the given line from the log.
*
* On error, NULL is returned.
* On success or soft ignores, GLogItem is returned. */
static GLogItem *
read_line (GLog *glog, char *line, int *test, uint32_t *cnt, int dry_run) {
GLogItem *logitem = NULL;
int ret = 0;
if ((ret = parse_line (glog, line, dry_run, &logitem)) == 0)
*test = 0;
if (ret == -1)
return NULL;
/* Increment local counter (will be synchronized later) */
if (conf.num_tests && ++(*cnt) >= conf.num_tests && *test) {
uncount_processed (glog);
uncount_invalid (glog);
return NULL;
}
atomic_fetch_add (&glog->read, 1); // Only 2 arguments!
return logitem;
}
/* Parse chunk of lines to logitems */
static void *
read_lines_thread (void *arg) {
GJob *job = (GJob *) arg;
int i = 0;
uint32_t local_cnt = atomic_load (&job->cnt);
for (i = 0; i < job->p; i++) {
/* Check stop_processing atomically */
if (atomic_load (&conf.stop_processing))
break;
/* ensure we don't process more than we should when testing for log format */
if (!job->test || (job->test && local_cnt < conf.num_tests)) {
job->logitems[i] = read_line (job->glog, job->lines[i], &job->test, &local_cnt, job->dry_run);
} else {
atomic_store (&conf.stop_processing, 1);
break;
}
#ifdef WITH_GETLINE
free (job->lines[i]);
#endif
}
/* Update shared counter atomically */
atomic_store (&job->cnt, local_cnt);
return (void *) 0;
}
/* A replacement for GNU getline() to dynamically expand fgets buffer.
* Wrapper for fgetline to work with GFileHandle abstraction
*
* On error, NULL is returned.
* On success, the malloc'd line is returned. */
char *
gfile_getline (GFileHandle *fh) {
char buf[LINE_BUFFER] = { 0 };
char *line = NULL, *tmp = NULL;
size_t linelen = 0, len = 0;
while (1) {
if (!gfile_gets (buf, sizeof (buf), fh)) {
if (conf.process_and_exit && errno == EAGAIN) {
(void) nanosleep ((const struct timespec[]) { {0, 100000000L} }, NULL);
continue;
}
/* Return partial line on EOF if we have data */
if (gfile_eof (fh) && line != NULL && linelen > 0)
return line;
break;
}
len = strlen (buf);
/* Skip empty reads but don't fail - could be transient. This is normal
* when tailing files */
if (len == 0)
continue;
/* overflow check */
if (SIZE_MAX - len - 1 < linelen)
break;
if ((tmp = realloc (line, linelen + len + 1)) == NULL)
break;
line = tmp;
strcpy (line + linelen, buf);
linelen += len;
/* Complete line or EOF reached */
if (gfile_eof (fh) || buf[len - 1] == '\n')
return line;
}
free (line);
return NULL;
}
void *
process_lines_thread (void *arg) {
GJob *job = (GJob *) arg;
int i = 0;
for (i = 0; i < job->p; i++) {
if (job->logitems[i] != NULL && !job->dry_run && job->logitems[i]->errstr == NULL) {
process_log (job->logitems[i]);
free_glog (job->logitems[i]);
job->logitems[i] = NULL;
}
}
return (void *) 0;
}
/* Initialize jobs */
static void
init_jobs (GJob jobs[2][conf.jobs], GLog *glog, int dry_run, int test) {
int b = 0, k = 0;
#ifndef WITH_GETLINE
int i = 0;
#endif
/* Initialize error mutex once */
static int mutex_initialized = 0;
if (!mutex_initialized) {
pthread_mutex_init (&glog->error_mutex, NULL);
mutex_initialized = 1;
}
for (b = 0; b < 2; b++) {
for (k = 0; k < conf.jobs; k++) {
jobs[b][k].p = 0;
atomic_store (&jobs[b][k].cnt, 0); // Use atomic store
jobs[b][k].glog = glog;
jobs[b][k].test = test;
jobs[b][k].dry_run = dry_run;
jobs[b][k].running = 0;
jobs[b][k].logitems = xcalloc (conf.chunk_size, sizeof (GLogItem));
jobs[b][k].lines = xcalloc (conf.chunk_size, sizeof (char *));
#ifndef WITH_GETLINE
for (i = 0; i < conf.chunk_size; i++)
jobs[b][k].lines[i] = xcalloc (LINE_BUFFER, sizeof (char));
#endif
}
}
}
/* Read lines from file */
static void
read_lines_from_file (GFileHandle *fh, GLog *glog, GJob jobs[2][conf.jobs], int b, char **s) {
int k = 0;
for (k = 0; k < conf.jobs; k++) {
#ifdef WITH_GETLINE
while ((*s = gfile_getline (fh)) != NULL) {
jobs[b][k].lines[jobs[b][k].p] = *s;
#else
while ((*s = gfile_gets (jobs[b][k].lines[jobs[b][k].p], LINE_BUFFER, fh)) != NULL) {
#endif
glog->bytes += strlen (jobs[b][k].lines[jobs[b][k].p]);
if (++(jobs[b][k].p) >= conf.chunk_size)
break; // goto next chunk
}
}
}
/* Processes lines using threads from the GJob array, updating counters. */
static void
process_lines (GJob jobs[2][conf.jobs], uint32_t *cnt, int *test, int b) {
int k = 0;
for (k = 0; k < conf.jobs; k++) {
process_lines_thread (&jobs[b][k]);
/* If interrupted, free any remaining logitems that weren't processed */
if (atomic_load (&conf.stop_processing)) {
int i;
for (i = 0; i < jobs[b][k].p; i++) {
if (jobs[b][k].logitems[i] != NULL) {
free_glog (jobs[b][k].logitems[i]);
jobs[b][k].logitems[i] = NULL;
}
}
}
/* Read atomic counter */
*cnt += atomic_load (&jobs[b][k].cnt);
atomic_store (&jobs[b][k].cnt, 0);
*test &= jobs[b][k].test;
jobs[b][k].p = 0;
}
}
/* Frees memory for lines and logitems in each job of the GJob array. */
static void
free_jobs (GJob jobs[2][conf.jobs]) {
int b = 0, k = 0;
#ifndef WITH_GETLINE
int i = 0;
#endif
for (b = 0; b < 2; b++) {
for (k = 0; k < conf.jobs; k++) {
#ifndef WITH_GETLINE
for (i = 0; i < conf.chunk_size; i++)
free (jobs[b][k].lines[i]);
#endif
free (jobs[b][k].logitems);
free (jobs[b][k].lines);
}
}
}
/* Reads lines from the given file pointer `fp` and processes them using
* parallel threads.
*
* On error or when interrupted by a signal (SIGINT), the function returns 0.
* On success, it returns 1 if the number of processed lines is greater than or
* equal to the configured number of tests (NUM_TESTS), otherwise 0.
*/
static int
read_lines (GFileHandle *fh, GLog *glog, int dry_run) {
int b = 0, k = 0, test = conf.num_tests > 0 ? 1 : 0;
uint32_t cnt = 0;
void *status = NULL;
char *s = NULL;
GJob jobs[2][conf.jobs];
pthread_t threads[conf.jobs];
glog->bytes = 0;
init_jobs (jobs, glog, dry_run, test);
b = 0;
while (1) { /* b = 0 or 1 */
read_lines_from_file (fh, glog, jobs, b, &s);
/* if nothing was read from the log, skip it for now */
if (!glog->bytes) {
test = 0;
break;
}
if (conf.jobs == 1) {
read_lines_thread (&jobs[b][0]);
} else {
for (k = 0; k < conf.jobs; k++) {
jobs[b][k].running = 1;
pthread_create (&threads[k], NULL, read_lines_thread, (void *) &jobs[b][k]);
}
}
/* flip from block A/B to B/A */
if (conf.jobs > 1)
b = b ^ 1;
process_lines (jobs, &cnt, &test, b);
/* flip from block B/A to A/B */
if (conf.jobs > 1)
b = b ^ 1;
if (conf.jobs > 1) {
for (k = 0; k < conf.jobs; k++) {
if (jobs[b][k].running) {
pthread_join (threads[k], &status);
jobs[b][k].running = 0;
}
}
}
if (dry_run && cnt >= NUM_TESTS)
break;
/* handle SIGINT */
if (conf.stop_processing)
break;
/* check for EOF */
if (s == NULL)
break;
/* flip from block A/B to B/A */
if (conf.jobs > 1)
b = b ^ 1;
} // while (1)
/* After eof, process last data */
for (b = 0; b < 2; b++) {
for (k = 0; k < conf.jobs; k++) {
if (conf.jobs > 1 && jobs[b][k].running) {
pthread_join (threads[k], &status);
jobs[b][k].running = 0;
}
if (jobs[b][k].p) {
process_lines_thread (&jobs[b][k]);
cnt += jobs[b][k].cnt;
jobs[b][k].cnt = 0;
test &= jobs[b][k].test;
jobs[b][k].p = 0;
}
}
}
free_jobs (jobs);
/* if no data was available to read from (probably from a pipe) and still in
* test mode and still below the test count, we simply return until data
* becomes available */
if (!s && (errno == EAGAIN || errno == EWOULDBLOCK) && test && cnt < conf.num_tests)
return 0;
return test;
}
/* Read and fingerprint the log prefix for file recognition on future runs.
*
* On error, 1 is returned.
* On success, 0 is returned. */
int
set_initial_persisted_data (GLog *glog, GFileHandle *fh, const char *fn) {
GLastParse lp = { 0 };
size_t len;
time_t now = time (0);
memset (glog->snippet, 0, sizeof (glog->snippet));
glog->snippetlen = 0;
glog->lp.snippetlen = 0;
glog->restore_snippetlen = 0;
memset (glog->lp.fingerprint, 0, sizeof (glog->lp.fingerprint));
if (glog->props.size == 0)
return 1;
len = MIN (glog->props.size, READ_BYTES);
if ((gfile_read (glog->snippet, len, 1, fh)) != 1 && gfile_error (fh))
FATAL ("Unable to read the specified log file '%s'", fn);
glog->snippetlen = len;
glog->lp.snippetlen = len;
sha256_digest (glog->snippet, len, glog->lp.fingerprint);
/* A file may have grown since a short prefix was saved. Hash that original
* length once here, rather than once per parsed line during restoration. */
lp = ht_get_last_parse (glog->props.inode);
if (lp.snippetlen && lp.snippetlen < len) {
glog->restore_snippetlen = lp.snippetlen;
sha256_digest (glog->snippet, lp.snippetlen, glog->restore_fingerprint);
}
localtime_r (&now, &glog->start_time);
gfile_seek (fh, 0, SEEK_SET);
return 0;
}
/* Store the position and fingerprint of the last parsed log. */
static void
persist_last_parse (GLog *glog) {
if (glog->props.inode && glog->props.size) {
glog->lp.line = glog->read;
ht_insert_last_parse (glog->props.inode, &glog->lp);
}
/* probably from a pipe */
else if (!glog->props.inode) {
ht_insert_last_parse (0, &glog->lp);
}
}
/* Read the given log line by line and process its data.
*
* On error, 1 is returned.
* On success, 0 is returned. */
static int
read_log (GLog *glog, int dry_run) {
GFileHandle *fh = NULL;
int piping = 0;
struct stat fdstat;
/* Reset stop_processing flag for new parse attempt */
atomic_store (&conf.stop_processing, 0);
/* Ensure we have a valid pipe to read from stdin. Only checking for
* conf.read_stdin without verifying for a valid FILE pointer would certainly
* lead to issues. */
if (glog->props.filename[0] == '-' && glog->props.filename[1] == '\0' && glog->pipe) {
/* For stdin pipe, we need to wrap the FILE* into a GFileHandle */
fh = calloc (1, sizeof (GFileHandle));
if (!fh)
FATAL ("Unable to allocate memory for file handle");
fh->fp = glog->pipe;
#ifdef HAVE_ZLIB
fh->is_gzipped = 0;
fh->gzfp = NULL;
#endif
glog->piping = piping = 1;
}
/* make sure we can open the log (if not reading from stdin) */
else if (!piping && (fh = gfile_open (glog->props.filename, "r")) == NULL)
FATAL ("Unable to open the specified log file '%s'. %s", glog->props.filename,
strerror (errno));
/* grab the inode of the file being parsed */
if (!piping && stat (glog->props.filename, &fdstat) == 0) {
glog->props.inode = fdstat.st_ino;
glog->props.size = glog->lp.size = fdstat.st_size;
set_initial_persisted_data (glog, fh, glog->props.filename);
}
/* read line by line */
if (read_lines (fh, glog, dry_run)) {
if (!piping)
gfile_close (fh);
else
free (fh); /* Just free the wrapper, don't close the pipe */
return 1;
}
persist_last_parse (glog);
/* close log file if not a pipe */
if (!piping)
gfile_close (fh);
else
free (fh); /* Just free the wrapper, don't close the pipe */
return 0;
}
static void
set_log_processing (Logs *logs, GLog *glog) {
lock_spinner ();
logs->processed = &(glog->processed);
logs->filename = glog->props.filename;
unlock_spinner ();
}
/* Entry point to parse the log line by line.
*
* On error, 1 is returned.
* On success, 0 is returned. */
int
parse_log (Logs *logs, int dry_run) {
GLog *glog = NULL;
const char *err_log = NULL;
int idx;
/* verify that we have the required formats */
if ((err_log = verify_formats ()))
FATAL ("%s", err_log);
/* no data piped, no logs passed, load from disk only then */
if (conf.restore && !logs->restored)
logs->restored = rebuild_rawdata_cache ();
/* no data piped, no logs passed, load from disk only then */
if (conf.restore && !conf.filenames_idx && !conf.read_stdin) {
logs->load_from_disk_only = 1;
return 0;
}
for (idx = 0; idx < logs->size; ++idx) {
glog = &logs->glog[idx];
set_log_processing (logs, glog);
if (read_log (glog, dry_run))
return 1;
glog->length = glog->bytes;
}
return 0;
}
/* Ensure we have valid hits
*
* On error, an array of pointers containing the error strings.
* On success, NULL is returned. */
char **
test_format (Logs *logs, int *len) {
char **errors = NULL;
GLog *glog = NULL;
int i;
if (parse_log (logs, 1) == 0)
return NULL;
for (i = 0; i < logs->size; ++i) {
glog = &logs->glog[i];
if (!glog->log_erridx)
continue;
break;
}
errors = xcalloc (glog->log_erridx, sizeof (char *));
*len = glog->log_erridx;
for (i = 0; i < glog->log_erridx; ++i)
errors[i] = xstrdup (glog->errors[i]);
free_logerrors (glog);
return errors;
}