added statistics db
sophuwu sophie@sophuwu.com
Mon, 28 Jul 2025 04:41:51 +0200
7 files changed,
300 insertions(+),
1 deletions(-)
M
embeds/embeds.go
→
embeds/embeds.go
@@ -7,6 +7,7 @@ "fmt"
"git.sophuwu.com/manweb/CFG" "git.sophuwu.com/manweb/logs" "git.sophuwu.com/manweb/neterr" + "git.sophuwu.com/manweb/stats" "html/template" "io/fs" "net/http"@@ -22,6 +23,9 @@ var help string
//go:embed template/login.html var LoginPage string + +//go:embed template/stats.html +var statsPage string //go:embed static/* var static embed.FS@@ -78,6 +82,7 @@ Query string
} func OpenAndParse() { + stats.T = template.Must(template.New("stats").Parse(statsPage)) openStatic() var e error t, e = template.New("index.html").Parse(index)@@ -111,6 +116,7 @@ Content: template.HTML(err.Error().Content()),
Query: q, } t.ExecuteTemplate(w, "index.html", p) + stats.SpecialCount("Error") } func WriteHtml(w http.ResponseWriter, r *http.Request, title, html string, q string, setRawQuery ...string) {
M
embeds/static/theme.css
→
embeds/static/theme.css
@@ -279,3 +279,45 @@ margin: 0;
padding: 0; height: 1.25lh; } +.stats span { + margin-top: 1ch; +} +.section-table { + display: flex; + flex-direction: row; + width: 100%; +} +.section-row { + display: flex; + flex-direction: column; +} +.section-row:first-child { + border-right: var(--fg-color) 1px solid; +} +.section-row:not(:first-child) > span:first-child { + font-weight: bold; +} +.section-row > span { + width: 10ch; + margin-left: 1ch; +} +.section-row:first-child > span { + margin-left: 0; +} +.query-table { + display: flex; + flex-direction: column; + width: fit-content; +} +.query-table > div { + display: flex; + flex-direction: row; + width: fit-content; +} +.query-table span { + width: 30ch; +} +.stats { + margin: 1ch; + padding: 1ch; +}
A
embeds/template/stats.html
@@ -0,0 +1,61 @@
+<br> +<h2> + Statistics +</h2> +<div class="stats"> + <h3> + Totals: + </h3> + <div class="query-table"> + <div> + <span>Number of queries:</span><span>{{ .TotalLoads }}</span> + </div> + <div> + <span>Total pages loaded:</span><span>{{ .TotalPages }}</span> + </div> + <div> + <span>Unique pages loaded:</span><span>{{ .UniquePages }}</span> + </div> + <div> + <span>Searches made:</span><span>{{ .Searches }}</span> + </div> + <div> + <span>Errors encountered:</span><span>{{ .Errors }}</span> + </div> + </div> +</div> +<div class="stats"> + <h3> + Pages loaded by section: + </h3> + <div class="section-table"> + <div class="section-row"> + <span>Section</span> + <span>Total</span> + <span>Unique</span> + </div> + {{ range .TotalSections }} + <div class="section-row"> + <span>{{ .Section }}</span> + <span>{{ .Count }}</span> + <span>{{ .Unique }}</span> + </div> + {{ end }} + </div> +</div> +<div class="stats"> + <h3> + Most queried pages: + </h3> + <style> + + </style> + <div class="query-table"> + {{ range .AllPages }} + <div> + <span><a href="?{{ .Page }}">{{ .Page }}</a></span> + <span>{{ .Count }}</span> + </div> + {{ end }} + </div> +</div>
M
main.go
→
main.go
@@ -3,6 +3,7 @@
import ( "fmt" "git.sophuwu.com/authuwu" + cookies "git.sophuwu.com/authuwu/cookie" "git.sophuwu.com/authuwu/userpass" "git.sophuwu.com/gophuwu/flags" "git.sophuwu.com/manweb/CFG"@@ -10,6 +11,7 @@ "git.sophuwu.com/manweb/embeds"
"git.sophuwu.com/manweb/logs" "git.sophuwu.com/manweb/manpage" "git.sophuwu.com/manweb/neterr" + "git.sophuwu.com/manweb/stats" "git.sophuwu.com/manweb/tldr" "golang.org/x/term" "net/http"@@ -96,8 +98,13 @@ }
if CFG.RequireAuth { err = authuwu.OpenDB(CFG.PasswdFile) logs.CheckFatal("opening password database", err) + _ = cookies.PurgeExpiredCookies() PageHandler = authuwu.NewAuthuwuHandler(PageHandler, time.Hour*24*3, embeds.LoginPage) defer authuwu.CloseDB() + } + if CFG.EnableStats { + stats.OpenDB() + defer stats.CloseDB() } CFG.ListenAndServe(Handler) }@@ -130,6 +137,7 @@ output += fmt.Sprintf(`<p><a href="?%s.%s">%s (%s)</a> - %s</p>%c`, line[1], line[2], line[1], line[2], line[3], 10)
} } embeds.WriteHtml(w, r, "Search", output, q, "") + stats.SpecialCount("Search") } var PageHandler http.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {@@ -147,7 +155,7 @@ }
} } if name == "" { - embeds.WriteHtml(w, r, "Index", "", "", "") + embeds.WriteHtml(w, r, "Index", stats.Html(), "", "") return } if embeds.Help(w, r, name) {
M
manpage/manpage.go
→
manpage/manpage.go
@@ -5,6 +5,7 @@ "fmt"
"git.sophuwu.com/manweb/CFG" "git.sophuwu.com/manweb/embeds" "git.sophuwu.com/manweb/neterr" + "git.sophuwu.com/manweb/stats" "net/http" "os/exec" "path/filepath"@@ -82,5 +83,6 @@ if embeds.ChkWriteError(w, r, err, q) {
return true } embeds.WriteHtml(w, r, m.Title(), html, q, m.Url()) + stats.Count(m.Url()) return true }
A
stats/stats.go
@@ -0,0 +1,178 @@
+package stats + +import ( + "bytes" + "git.sophuwu.com/manweb/CFG" + "git.sophuwu.com/manweb/logs" + "github.com/asdine/storm/v3" + "go.etcd.io/bbolt" + "html/template" + "path/filepath" + "strings" +) + +var DB *storm.DB + +func OpenDB() { + if !CFG.EnableStats || CFG.StatisticDB == "" { + return + } + var err error + DB, err = storm.Open(CFG.StatisticDB, storm.BoltOptions(0660, &bbolt.Options{ + Timeout: 1000 * 1000 * 1000, // 1 second + })) + logs.CheckFatal("failed to open statistics database", err) +} + +func CloseDB() { + if DB == nil { + return + } + err := DB.Close() + logs.CheckFatal("failed to close statistics database", err) + DB = nil +} + +type Stat struct { + Query string `storm:"id,unique"` // The query string + Count int +} + +func count(query string) { + s := &Stat{Query: query} + err := DB.One("Query", query, s) + if err != nil { + s = &Stat{ + Query: query, + Count: 1, + } + DB.Save(s) + return + } + s.Count++ + DB.Update(s) +} + +func Count(query string) { + if !CFG.EnableStats || DB == nil { + return + } + go count(query) +} + +type Special struct { + Page string `storm:"id,unique"` // The page name + Count int +} + +func countSpecial(page string) { + s := &Special{Page: page} + err := DB.One("Page", page, s) + if err != nil { + s = &Special{ + Page: page, + Count: 1, + } + DB.Save(s) + return + } + s.Count++ + DB.Update(s) +} + +func SpecialCount(page string) { + if !CFG.EnableStats || DB == nil { + return + } + go countSpecial(page) +} + +func GetStats() ([]*Stat, error) { + var stats []*Stat + err := DB.Select().OrderBy("Count").Reverse().Find(&stats) + if err != nil { + return nil, err + } + return stats, nil +} + +type SectionCount struct { + Section string + Count int + Unique int +} + +func addSection(sec *map[string]*SectionCount, name string, count int) { + if name == "" { + return + } + name = filepath.Ext(name) + name = strings.TrimPrefix(name, ".") + secName, ok := (*sec)[name] + if !ok { + (*sec)[name] = &SectionCount{ + Section: name, + Count: count, + Unique: 1, + } + return + } + secName.Count += count + secName.Unique++ + (*sec)[name] = secName +} + +type html struct { + TotalLoads int + TotalPages int + UniquePages int + Searches int + Errors int + TotalSections []SectionCount + AllPages []Special + MaxLen int +} + +var T *template.Template + +func Html() string { + if !CFG.EnableStats || DB == nil { + return "" + } + stats, err := GetStats() + if err != nil || len(stats) == 0 { + return "" + } + var ht html + var Specialc Special + _ = DB.One("Page", "Search", &Specialc) + ht.Searches = Specialc.Count + _ = DB.One("Page", "Error", &Specialc) + ht.Errors = Specialc.Count + + sec := make(map[string]*SectionCount) + Maxlen := 0 + for _, s := range stats { + addSection(&sec, s.Query, s.Count) + ht.AllPages = append(ht.AllPages, Special{Page: s.Query, Count: s.Count}) + if len(s.Query) > Maxlen { + Maxlen = len(s.Query) + } + } + for _, v := range sec { + ht.TotalSections = append(ht.TotalSections, *v) + ht.TotalPages += v.Count + ht.UniquePages += v.Unique + } + for i := range ht.TotalSections { + for j := i + 1; j < len(ht.TotalSections); j++ { + if ht.TotalSections[i].Count > ht.TotalSections[j].Count { + ht.TotalSections[i], ht.TotalSections[j] = ht.TotalSections[j], ht.TotalSections[i] + } + } + } + ht.TotalLoads = ht.TotalPages + ht.UniquePages + ht.Searches + ht.Errors + var b bytes.Buffer + _ = T.ExecuteTemplate(&b, "stats", &ht) + return b.String() +}
M
tldr/tldr.go
→
tldr/tldr.go
@@ -6,6 +6,7 @@ "git.sophuwu.com/manweb/CFG"
"git.sophuwu.com/manweb/embeds" "git.sophuwu.com/manweb/logs" "git.sophuwu.com/manweb/neterr" + "git.sophuwu.com/manweb/stats" "net/http" "os" "strings"@@ -222,5 +223,6 @@ embeds.WriteError(w, r, neterr.Err500, q)
return true } embeds.WriteHtml(w, r, "TLDR: "+name, html, q, q) + stats.Count(q) return true }