Commit 8f582c25 authored by Christian Blechert's avatar Christian Blechert

initial commit

parents
Pipeline #5 failed with stages
## Ignore Visual Studio temporary files, build results, and
## files generated by popular Visual Studio add-ons.
# User-specific files
*.suo
*.user
*.userosscache
*.sln.docstates
# User-specific files (MonoDevelop/Xamarin Studio)
*.userprefs
# Build results
[Dd]ebug/
[Dd]ebugPublic/
[Rr]elease/
[Rr]eleases/
x64/
x86/
bld/
[Bb]in/
[Oo]bj/
[Ll]og/
# Visual Studio 2015 cache/options directory
.vs/
# Uncomment if you have tasks that create the project's static files in wwwroot
#wwwroot/
# MSTest test Results
[Tt]est[Rr]esult*/
[Bb]uild[Ll]og.*
# NUNIT
*.VisualState.xml
TestResult.xml
# Build Results of an ATL Project
[Dd]ebugPS/
[Rr]eleasePS/
dlldata.c
# DNX
project.lock.json
artifacts/
*_i.c
*_p.c
*_i.h
*.ilk
*.meta
*.obj
*.pch
*.pdb
*.pgc
*.pgd
*.rsp
*.sbr
*.tlb
*.tli
*.tlh
*.tmp
*.tmp_proj
*.log
*.vspscc
*.vssscc
.builds
*.pidb
*.svclog
*.scc
# Chutzpah Test files
_Chutzpah*
# Visual C++ cache files
ipch/
*.aps
*.ncb
*.opendb
*.opensdf
*.sdf
*.cachefile
# Visual Studio profiler
*.psess
*.vsp
*.vspx
*.sap
# TFS 2012 Local Workspace
$tf/
# Guidance Automation Toolkit
*.gpState
# ReSharper is a .NET coding add-in
_ReSharper*/
*.[Rr]e[Ss]harper
*.DotSettings.user
# JustCode is a .NET coding add-in
.JustCode
# TeamCity is a build add-in
_TeamCity*
# DotCover is a Code Coverage Tool
*.dotCover
# NCrunch
_NCrunch_*
.*crunch*.local.xml
nCrunchTemp_*
# MightyMoose
*.mm.*
AutoTest.Net/
# Web workbench (sass)
.sass-cache/
# Installshield output folder
[Ee]xpress/
# DocProject is a documentation generator add-in
DocProject/buildhelp/
DocProject/Help/*.HxT
DocProject/Help/*.HxC
DocProject/Help/*.hhc
DocProject/Help/*.hhk
DocProject/Help/*.hhp
DocProject/Help/Html2
DocProject/Help/html
# Click-Once directory
publish/
# Publish Web Output
*.[Pp]ublish.xml
*.azurePubxml
# TODO: Comment the next line if you want to checkin your web deploy settings
# but database connection strings (with potential passwords) will be unencrypted
*.pubxml
*.publishproj
# NuGet Packages
*.nupkg
# The packages folder can be ignored because of Package Restore
**/packages/*
# except build/, which is used as an MSBuild target.
!**/packages/build/
# Uncomment if necessary however generally it will be regenerated when needed
#!**/packages/repositories.config
# NuGet v3's project.json files produces more ignoreable files
*.nuget.props
*.nuget.targets
# Microsoft Azure Build Output
csx/
*.build.csdef
# Microsoft Azure Emulator
ecf/
rcf/
# Microsoft Azure ApplicationInsights config file
ApplicationInsights.config
# Windows Store app package directory
AppPackages/
BundleArtifacts/
# Visual Studio cache files
# files ending in .cache can be ignored
*.[Cc]ache
# but keep track of directories ending in .cache
!*.[Cc]ache/
# Others
ClientBin/
~$*
*~
*.dbmdl
*.dbproj.schemaview
*.pfx
*.publishsettings
node_modules/
orleans.codegen.cs
# RIA/Silverlight projects
Generated_Code/
# Backup & report files from converting an old project file
# to a newer Visual Studio version. Backup files are not needed,
# because we have git ;-)
_UpgradeReport_Files/
Backup*/
UpgradeLog*.XML
UpgradeLog*.htm
# SQL Server files
*.mdf
*.ldf
# Business Intelligence projects
*.rdl.data
*.bim.layout
*.bim_*.settings
# Microsoft Fakes
FakesAssemblies/
# GhostDoc plugin setting file
*.GhostDoc.xml
# Node.js Tools for Visual Studio
.ntvs_analysis.dat
# Visual Studio 6 build log
*.plg
# Visual Studio 6 workspace options file
*.opt
# Visual Studio LightSwitch build output
**/*.HTMLClient/GeneratedArtifacts
**/*.DesktopClient/GeneratedArtifacts
**/*.DesktopClient/ModelManifest.xml
**/*.Server/GeneratedArtifacts
**/*.Server/ModelManifest.xml
_Pvt_Extensions
# Paket dependency manager
.paket/paket.exe
# FAKE - F# Make
.fake/
{
// Use IntelliSense to learn about possible attributes.
// Hover to view descriptions of existing attributes.
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
"version": "0.2.0",
"configurations": [
{
"name": ".NET Core Launch (console)",
"type": "coreclr",
"request": "launch",
"preLaunchTask": "build",
"program": "${workspaceFolder}/SteamCacheStatsCore/bin/Debug/netcoreapp2.1/SteamCacheStatsCore.dll",
"args": [
"/home/christian/Documents/lidl/cachelog",
"*.log",
"/home/christian/Documents/lidl/out.html"
],
"cwd": "${workspaceFolder}/SteamCacheStatsCore",
"console": "externalTerminal",
"stopAtEntry": false,
"internalConsoleOptions": "openOnSessionStart"
},
{
"name": ".NET Core Attach",
"type": "coreclr",
"request": "attach",
"processId": "${command:pickProcess}"
}
]
}
\ No newline at end of file
{
"version": "2.0.0",
"tasks": [
{
"label": "build",
"command": "dotnet",
"type": "process",
"args": [
"build",
"${workspaceFolder}/SteamCacheStatsCore/SteamCacheStatsCore.csproj"
],
"problemMatcher": "$msCompile"
}
]
}
\ No newline at end of file
Copyright (c) 2018 Christian Blechert
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is furnished
to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
\ No newline at end of file
Parse the access logs of a steamcache installation and create traffic statistics.
## Usage
```
./SteamCacheStats "/path/to/log/dir" "*.log" "/output/htmlfile.html"
```
Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio 15
VisualStudioVersion = 15.0.28010.2050
MinimumVisualStudioVersion = 10.0.40219.1
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SteamCacheStatsCore", "SteamCacheStatsCore\SteamCacheStatsCore.csproj", "{23A99EA7-A22A-463C-8EFC-A1F25E84CDFB}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Debug|x64 = Debug|x64
Debug|x86 = Debug|x86
Release|Any CPU = Release|Any CPU
Release|x64 = Release|x64
Release|x86 = Release|x86
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{23A99EA7-A22A-463C-8EFC-A1F25E84CDFB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{23A99EA7-A22A-463C-8EFC-A1F25E84CDFB}.Debug|Any CPU.Build.0 = Debug|Any CPU
{23A99EA7-A22A-463C-8EFC-A1F25E84CDFB}.Debug|x64.ActiveCfg = Debug|Any CPU
{23A99EA7-A22A-463C-8EFC-A1F25E84CDFB}.Debug|x64.Build.0 = Debug|Any CPU
{23A99EA7-A22A-463C-8EFC-A1F25E84CDFB}.Debug|x86.ActiveCfg = Debug|Any CPU
{23A99EA7-A22A-463C-8EFC-A1F25E84CDFB}.Debug|x86.Build.0 = Debug|Any CPU
{23A99EA7-A22A-463C-8EFC-A1F25E84CDFB}.Release|Any CPU.ActiveCfg = Release|Any CPU
{23A99EA7-A22A-463C-8EFC-A1F25E84CDFB}.Release|Any CPU.Build.0 = Release|Any CPU
{23A99EA7-A22A-463C-8EFC-A1F25E84CDFB}.Release|x64.ActiveCfg = Release|Any CPU
{23A99EA7-A22A-463C-8EFC-A1F25E84CDFB}.Release|x64.Build.0 = Release|Any CPU
{23A99EA7-A22A-463C-8EFC-A1F25E84CDFB}.Release|x86.ActiveCfg = Release|Any CPU
{23A99EA7-A22A-463C-8EFC-A1F25E84CDFB}.Release|x86.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {8FF77064-7FD8-48B9-AB26-8894BA5796EB}
EndGlobalSection
EndGlobal
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
namespace SteamCacheStats.Data
{
public enum CacheStatus { HIT, MISS }
public class LogEntry
{
// $remote_addr / $http_x_forwarded_for - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" "$upstream_cache_status" "$host" "$http_range"
// 172.23.17.93 / - - - [10/Nov/2018:02:42:56 +0000] "GET /latest64/101 HTTP/1.1" 200 36 "-" "-" "HIT" "assetcdn.101.arenanetworks.com" "-"
public static string LINERGXSTR { get { return "^(?<clientip>[^\\s]+) / [^\\[]+\\[(?<timestamp>[^\\]]+)\\] \"(?<request>[^\"]+)\" (?<statuscode>[0-9]+) (?<bytes>[0-9]+) \".+?\" \".+?\" \"(?<cachestatus>HIT|MISS)\" \"(?<remoteserver>[^\"]+)\" \".+?\"$"; } }
public static string DATEFORMAT { get { return "dd/MMM/yyyy:HH:mm:ss"; } }
protected static Regex Regex { get; set; } = new Regex(LINERGXSTR, RegexOptions.Compiled);
public string Filename { get; set; }
public DateTime Timestamp { get; set; }
public string ClientIP { get; set; }
public string Request { get; set; }
public uint StatusCode { get; set; }
public ulong Bytes { get; set; }
public CacheStatus CachingStatus { get; set; }
public string RemoteHost { get; set; }
public static bool TryParse(string logline, ref LogEntry entry)
{
if (string.IsNullOrEmpty(logline) == false && Regex.IsMatch(logline))
{
var match = Regex.Match(logline);
entry = FromRegex(match);
return true;
}
return false;
}
public static LogEntry FromRegex(Match match)
{
return new LogEntry()
{
ClientIP = match.Groups["clientip"].Value,
Timestamp = ParseDate(match.Groups["timestamp"].Value),
Request = match.Groups["request"].Value,
StatusCode = uint.Parse(match.Groups["statuscode"].Value),
Bytes = ulong.Parse(match.Groups["bytes"].Value),
CachingStatus = GetCacheStatus(match.Groups["cachestatus"].Value),
RemoteHost = match.Groups["remoteserver"].Value
};
}
public static CacheStatus GetCacheStatus(string status)
{
if(status=="HIT")
{
return CacheStatus.HIT;
}
else if(status=="MISS")
{
return CacheStatus.MISS;
}
throw new ArgumentException("Invalid cache status");
}
protected static DateTime ParseDate(string s)
{
var temp = s.Substring(0, 20);
return DateTime.ParseExact(temp, DATEFORMAT, CultureInfo.InvariantCulture);
}
}
}
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
namespace SteamCacheStats.Data
{
class LogParser
{
public LogParser()
{ }
public ParserResult Parse(string filename, TextReader reader)
{
var result = new ParserResult(filename);
string line;
while ((line = reader.ReadLine()) != null)
{
LogEntry entry = null;
if (LogEntry.TryParse(line, ref entry))
{
if (entry.StatusCode == 200 && entry.Bytes > 0)
{
result.Commit(entry);
}
}
}
return result;
}
public async Task<ParserResult> ParseAsync(string filename)
{
return await Task.Run(() =>
{
using (Stream stream = new FileStream(filename, FileMode.Open, FileAccess.Read, FileShare.Read))
using (TextReader reader = new StreamReader(stream, Encoding.UTF8))
{
return this.Parse(filename, reader);
}
});
}
}
}
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace SteamCacheStats.Data
{
public class ParserResult
{
public string LogFile { get; set; }
public ulong TotalRequests { get; set; }
public ulong FetchedRequests { get; set; }
public ulong CachedRequests { get; set; }
public ulong TotalBytes { get; set; }
public ulong CachedBytes { get; set; }
public ulong FetchedBytes { get; set; }
public DateTime? FirstRequest { get; set; }
public DateTime? LastRequest { get; set; }
public ParserResult(string logfile)
{
this.LogFile = logfile;
}
public void Commit(LogEntry entry)
{
this.TotalRequests++;
this.TotalBytes += entry.Bytes;
if (entry.CachingStatus == CacheStatus.HIT)
{
this.CachedBytes += entry.Bytes;
this.CachedRequests++;
}
else
{
this.FetchedBytes += entry.Bytes;
this.FetchedRequests++;
}
if (this.LastRequest == null || entry.Timestamp > this.LastRequest)
{
this.LastRequest = entry.Timestamp;
}
if (this.FirstRequest == null || entry.Timestamp < this.FirstRequest)
{
this.FirstRequest = entry.Timestamp;
}
}
/*
public ParserResult(string logfile, IEnumerable<LogEntry> entries)
{
this.LogFile = logfile;
this.Process(entries);
}
public async void Process(IEnumerable<LogEntry> entries)
{
await Task.Run(() =>
{
this.TotalRequests = (long)entries.Count();
this.FetchedRequests = (long)entries.Where(v => v.CachingStatus == "MISS").Count();
this.CachedRequests = (long)entries.Where(v => v.CachingStatus == "HIT").Count();
this.TotalBytes = entries.Sum(v => v.Bytes);
this.CachedBytes = entries.Where(v => v.CachingStatus == "HIT").Sum(v => v.Bytes);
this.FetchedBytes = entries.Where(v => v.CachingStatus == "MISS").Sum(v => v.Bytes);
this.FirstRequest = entries.Count() > 0 ? entries.Min(v => v.Timestamp) : (DateTime?)null;
this.LastRequest = entries.Count() > 0 ? entries.Max(v => v.Timestamp) : (DateTime?)null;
});
}
*/
}
}
using PerrysNetConsole;
using SteamCacheStats.Data;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Text;
using System.Threading.Tasks;
namespace SteamCacheStatsCore
{
class Program
{
public static void Main(string[] args)
{
CoEx.ForcedBufferWidth = 120;
CoEx.TrySetWindowSize(121, 40);
if (!(args.Count() == 3 && Directory.Exists(args[0]) && Directory.Exists(Path.GetDirectoryName(args[2]))))
{
CoEx.WriteLine("Usage: ProgramExecutable directory_of_logfiles *.log targetfile.html");
Environment.Exit(1);
return;
}
var path = Path.GetDirectoryName(Uri.UnescapeDataString((new UriBuilder(Assembly.GetExecutingAssembly().CodeBase)).Path));
using (var writer = new PerrysNetConsoleHtml.CoExHtmlWriter(new FileInfo(Path.Combine(path, "terminal.html")), Encoding.UTF8))
{
CoEx.WriteTitleLarge("Game Cache Statistics (Just parsed the access logs...)");
CoEx.WriteLine();
writer.Pause();
LoadIndicator load = new LoadIndicator() { Message = "Analyze..." };
load.Start();
// Parse logfiles
LogParser parser = new LogParser();
var tasks = new List<Task<ParserResult>>();
foreach (var logfile in Directory.GetFiles(args[0], args[1]))
{
tasks.Add(parser.ParseAsync(logfile));
}
var taskarray = tasks.ToArray();
Task.WaitAll(taskarray);
load.Stop();
writer.Resume();
// Table by platform
var result = taskarray
.Select(v => v.Result)
.OrderByDescending(v => v.TotalBytes)
.Select(v => new string[]
{
Path.GetFileName(v.LogFile).Substring(0, Path.GetFileName(v.LogFile).IndexOf('-')),
$"{((0.0+v.TotalBytes)/1024/1024).ToString("#,##0")} MB",
$"{((0.0+v.FetchedBytes)/1024/1024).ToString("#,##0")} MB",
$"{((0.0+v.CachedBytes)/1024/1024).ToString("#,##0")} MB",
$"{(v.TotalBytes>0?(100.0/v.TotalBytes*v.CachedBytes):0).ToString("0.00")} %",
$"{v.TotalRequests.ToString("#,##0")}",
v.FirstRequest?.ToString("yyyy-MM-dd HH:mm")??"-",
v.LastRequest?.ToString("yyyy-MM-dd HH:mm")??"-"
})
.ToList();
var rows = RowCollection.Create(result.ToArray());
rows.Import(0, RowConf.Create(new string[]
{
"File",
"Total Bytes",
"Fetched Bytes",
"Cached Bytes",
"Cached %",
"Requests",
"First",
"Last"
})
.PresetTH());
rows.Settings.Border.Enabled = true;
rows.Settings.Align = (conf, idx, val) =>
{
switch (idx)
{
case 1:
case 2:
case 3:
case 4:
case 5:
return RowCollectionSettings.ALIGN.RIGHT;
default:
return RowCollectionSettings.ALIGN.LEFT;
}
};
CoEx.WriteTable(rows);
// Summary
ulong granttotalbytes = taskarray.Select(v => v.Result.TotalBytes).Aggregate((a, c) => a + c);
ulong grantcachedbytes = taskarray.Select(v => v.Result.CachedBytes).Aggregate((a, c) => a + c);
ulong granttotalrequests = taskarray.Select(v => v.Result.TotalRequests).Aggregate((a, c) => a + c);
ulong grantcachedrequests = taskarray.Select(v => v.Result.CachedRequests).Aggregate((a, c) => a + c);
CoEx.WriteLine();
CoEx.WriteLine($"Processed {(granttotalbytes / 1024 / 1024).ToString("#,##0")} MB of traffic");
CoEx.WriteLine($"Saved {(grantcachedbytes / 1024 / 1024).ToString("#,##0")} MB of traffic");
CoEx.WriteLine($"About {(100.0 / granttotalbytes * grantcachedbytes).ToString("0.00")}% of the traffic was saved");
CoEx.WriteLine();
CoEx.WriteLine($"Processed {granttotalrequests.ToString("#,##0")} Requests");
CoEx.WriteLine($"Saved {grantcachedrequests.ToString("#,##0")} Requests");
writer.SaveAs(args[2]);
}
}
}
}
{
"profiles": {
"SteamCacheStatsCore": {
"commandName": "Project",
"commandLineArgs": "\"C:\\Users\\christian.BRICKBURG\\Downloads\\cachelog\" \"*.log\" \"C:\\Users\\christian.BRICKBURG\\Downloads\\cachelog\\stats.html\""
}
}
}
\ No newline at end of file
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>netcoreapp2.1</TargetFramework>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="PerrysNetConsole" Version="2.0.0-alpha" />
<PackageReference Include="PerrysNetConsoleHtml" Version="2.0.0-alpha" />
</ItemGroup>
<ItemGroup>
<None Update="terminal.html">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
</ItemGroup>
</Project>
This source diff could not be displayed because it is too large. You can view the blob instead.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment