Change 'timestamp' to simple unix epoch

Subjectively, 6d4e7a96dc increases
lag on a years-old production database with ~240k entries when searching
with CTRL+R.

So change timestamp column from datetime type (which is a string)
to integer, storing unix epoch now. As we deal with those primarily,
we also avoid conversions now.

Mentioned database shrinks by 2MB compared to previously (both VACCUMed)...

DROP unused count_by_date VIEW.

README.md

@@ -2,15 +2,34 @@
 hs9001 (history search 9001) is an easy, quite simple bash history enhancement. It simply writes all
 your bash commands into an sqlite database. You can then search this database.
 
+It improves over bash's built-in history mechanism as it'll aggregate the shell history of all open shells,
+timestamp them and also record additional information such as the directory a command was executed in.
+
+## Usage / Examples
+### Search
+
+```
+hs [search terms]
+```
+You can further filter with options like `-cwd`, `-after` and so on...
+For a full list, see `-help`.
+
+```
+hs -cwd . 
+``` 
+Lists all commands ever entered in this directory
+
+```
+hs -today -cwd . git
+``` 
+Lists all git commands in the current directory which have been entered today.
+
+Also, it (by default) replaces bash's built-in CTRL-R mechanism, so hs9001's database will be used instead of bash's limited history files.
+
+When in reverse-search mode, you can only search the history of the current directory by pressing CTRL+A and then "w".
 
 ## Install
 
-### From source
-```
-go build
-#move hs9001 to a PATH location
-```
-
 ### Debian / Ubuntu
 Latest release can be installed using apt
 ```
@@ -28,31 +47,22 @@ apk update
 apk add hs9001
 ```
 
-
+### From source
-### Setup / Config
+```
-
+go build
+#move hs9001 to a PATH location
+```
 Add this to .bashrc
 
 ```
 eval "$(hs9001 bash-enable)"
 ```
-By default, every system user gets his own database. You can override this by setting the environment variable
-for all users that should write to your unified database.
 
+This will also create a `hs`alias so you have to type less in everyday usage.
+
+By default, every system user gets his own database. You can override this by setting the environment variable for all users that should write to your unified database.
 ```
 export HS9001_DB_PATH="/home/db/history.sqlite"
 ```
 
-## Usage
-### Search
-
-```
-hs9001 search [search terms]
-```
-
-It is recommended to create an alias for search to make life easier, e. g.:
-
-```
-alias searchh='hs9001 search'
-```
 

history.go

@@ -2,8 +2,10 @@ package main
 
 import (
 	"database/sql"
+	"hs9001/liner"
 	"io"
 	"log"
+	"path/filepath"
 	"strings"
 )
 
@@ -11,17 +13,34 @@ type history struct {
 	conn *sql.DB
 }
 
-func (h *history) GetHistoryByPrefix(prefix string) (ph []string) {
+func createSearchOpts(query string, mode int) searchopts {
-	/* Hack for performance reasons */
-	if len(prefix) < 2 {
-		return
-	}
 	opts := searchopts{}
-	opts.order = "DESC"
+	o := "DESC"
-	cmdqry := prefix + "%"
+	opts.order = &o
-	opts.command = &cmdqry
+	lim := 100
+	opts.limit = &lim
+	opts.command = &query
+
+	switch mode {
+	case liner.ModeGlobal:
+		break
+	case liner.ModeWorkdir:
+		workdir, err := filepath.Abs(".")
+		if err != nil {
+			panic(err)
+		}
+		opts.workdir = &workdir
+	default:
+		panic("Invalid mode supplied")
+	}
+	return opts
+}
+
+func (h *history) GetHistoryByPrefix(prefix string, mode int) (ph []string) {
+	cmdquery := prefix + "%"
+	opts := createSearchOpts(cmdquery, mode)
 	results := search(h.conn, opts)
-	for e := results.Front(); e != nil; e = e.Next() {
+	for e := results.Back(); e != nil; e = e.Prev() {
 		entry, ok := e.Value.(*HistoryEntry)
 		if !ok {
 			log.Panic("Failed to retrieve entries")
@@ -30,17 +49,13 @@ func (h *history) GetHistoryByPrefix(prefix string) (ph []string) {
 	}
 	return
 }
-func (h *history) GetHistoryByPattern(pattern string) (ph []string, pos []int) {
+
-	/* Hack for performance reasons */
+func (h *history) GetHistoryByPattern(pattern string, mode int) (ph []string, pos []int) {
-	if len(pattern) < 2 {
+	cmdquery := "%" + pattern + "%"
-		return
+	opts := createSearchOpts(cmdquery, mode)
-	}
+
-	opts := searchopts{}
-	opts.order = "DESC"
-	cmdqry := "%" + pattern + "%"
-	opts.command = &cmdqry
 	results := search(h.conn, opts)
-	for e := results.Front(); e != nil; e = e.Next() {
+	for e := results.Back(); e != nil; e = e.Prev() {
 		entry, ok := e.Value.(*HistoryEntry)
 		if !ok {
 			log.Panic("Failed to retrieve entries")

liner/common.go

@@ -37,8 +37,8 @@ type HistoryProvider interface {
 	WriteHistory(w io.Writer) (num int, err error)
 	AppendHistory(item string)
 	ClearHistory()
-	GetHistoryByPrefix(prefix string) (ph []string)
+	GetHistoryByPrefix(prefix string, mode int) (ph []string)
-	GetHistoryByPattern(pattern string) (ph []string, pos []int)
+	GetHistoryByPattern(pattern string, mode int) (ph []string, pos []int)
 	RLock()
 	RUnlock()
 }
@@ -95,11 +95,11 @@ func (s *State) AppendHistory(item string) {
 func (s *State) ClearHistory() {
 	s.historyProvider.ClearHistory()
 }
-func (s *State) getHistoryByPrefix(prefix string) (ph []string) {
+func (s *State) getHistoryByPrefix(prefix string, mode int) (ph []string) {
-	return s.historyProvider.GetHistoryByPrefix(prefix)
+	return s.historyProvider.GetHistoryByPrefix(prefix, mode)
 }
-func (s *State) getHistoryByPattern(pattern string) (ph []string, pos []int) {
+func (s *State) getHistoryByPattern(pattern string, mode int) (ph []string, pos []int) {
-	return s.historyProvider.GetHistoryByPattern(pattern)
+	return s.historyProvider.GetHistoryByPattern(pattern, mode)
 }
 
 // SetHistoryProvider allows you to set a custom provider

liner/line.go

@@ -90,6 +90,11 @@ const (
 	tabReverse
 )
 
+const (
+	ModeGlobal = iota
+	ModeWorkdir
+)
+
 func (s *State) refresh(prompt []rune, buf []rune, pos int) error {
 	if s.columns == 0 {
 		return ErrInternal
@@ -411,8 +416,26 @@ func (s *State) tabComplete(p []rune, line []rune, pos int) ([]rune, int, interf
 
 // reverse intelligent search, implements a bash-like history search.
 func (s *State) reverseISearch(origLine []rune, origPos int) ([]rune, int, interface{}, error) {
-	p := "(reverse-i-search)`': "
+	modeSelect := false
-	err := s.refresh([]rune(p), origLine, origPos)
+	currentMode := ModeGlobal
+
+	getPrompt := func(arg string) string {
+		prompt := ""
+		switch currentMode {
+		case ModeWorkdir:
+			prompt = "(reverse:cwd)`%s': "
+		case ModeGlobal:
+			prompt = "(reverse:global)`%s': "
+		default:
+			panic("Invalid mode")
+		}
+		if modeSelect {
+			prompt = "(select mode)`%s': "
+		}
+		return fmt.Sprintf(prompt, arg)
+	}
+
+	err := s.refresh([]rune(getPrompt("")), origLine, origPos)
 	if err != nil {
 		return origLine, origPos, rune(esc), err
 	}
@@ -426,11 +449,10 @@ func (s *State) reverseISearch(origLine []rune, origPos int) ([]rune, int, inter
 
 	getLine := func() ([]rune, []rune, int) {
 		search := string(line)
-		prompt := "(reverse-i-search)`%s': "
+		return []rune(getPrompt(search)), []rune(foundLine), foundPos
-		return []rune(fmt.Sprintf(prompt, search)), []rune(foundLine), foundPos
 	}
 
-	history, positions := s.getHistoryByPattern(string(line))
+	history, positions := s.getHistoryByPattern(string(line), currentMode)
 	historyPos := len(history) - 1
 
 	for {
@@ -450,6 +472,8 @@ func (s *State) reverseISearch(origLine []rune, origPos int) ([]rune, int, inter
 				} else {
 					s.doBeep()
 				}
+			case ctrlA:
+				modeSelect = true
 			case ctrlS: // Search forward
 				if historyPos < len(history)-1 && historyPos >= 0 {
 					historyPos++
@@ -467,7 +491,7 @@ func (s *State) reverseISearch(origLine []rune, origPos int) ([]rune, int, inter
 					pos -= n
 
 					// For each char deleted, display the last matching line of history
-					history, positions := s.getHistoryByPattern(string(line))
+					history, positions := s.getHistoryByPattern(string(line), currentMode)
 					historyPos = len(history) - 1
 					if len(history) > 0 {
 						foundLine = history[historyPos]
@@ -480,17 +504,28 @@ func (s *State) reverseISearch(origLine []rune, origPos int) ([]rune, int, inter
 			case ctrlG: // Cancel
 				return origLine, origPos, rune(esc), err
 
-			case tab, cr, lf, ctrlA, ctrlB, ctrlD, ctrlE, ctrlF, ctrlK,
+			case tab, cr, lf, ctrlB, ctrlD, ctrlE, ctrlF, ctrlK,
 				ctrlL, ctrlN, ctrlO, ctrlP, ctrlQ, ctrlT, ctrlU, ctrlV, ctrlW, ctrlX, ctrlY, ctrlZ:
 				fallthrough
 			case 0, ctrlC, esc, 28, 29, 30, 31:
 				return []rune(foundLine), foundPos, next, err
 			default:
+				if modeSelect {
+					switch v {
+					case 'g':
+						currentMode = ModeGlobal
+					case 'w':
+						currentMode = ModeWorkdir
+					}
+					modeSelect = false
+					break
+				}
+
 				line = append(line[:pos], append([]rune{v}, line[pos:]...)...)
 				pos++
 
 				// For each keystroke typed, display the last matching line of history
-				history, positions = s.getHistoryByPattern(string(line))
+				history, positions = s.getHistoryByPattern(string(line), currentMode)
 				historyPos = len(history) - 1
 				if len(history) > 0 {
 					foundLine = history[historyPos]
@@ -726,7 +761,7 @@ mainLoop:
 			case ctrlP: // up
 				historyAction = true
 				if historyStale {
-					historyPrefix = s.getHistoryByPrefix(string(line))
+					historyPrefix = s.getHistoryByPrefix(string(line), ModeGlobal)
 					historyPos = len(historyPrefix)
 					historyStale = false
 				}
@@ -744,7 +779,7 @@ mainLoop:
 			case ctrlN: // down
 				historyAction = true
 				if historyStale {
-					historyPrefix = s.getHistoryByPrefix(string(line))
+					historyPrefix = s.getHistoryByPrefix(string(line), ModeGlobal)
 					historyPos = len(historyPrefix)
 					historyStale = false
 				}
@@ -912,7 +947,7 @@ mainLoop:
 			case up:
 				historyAction = true
 				if historyStale {
-					historyPrefix = s.getHistoryByPrefix(string(line))
+					historyPrefix = s.getHistoryByPrefix(string(line), ModeGlobal)
 					historyPos = len(historyPrefix)
 					historyStale = false
 				}
@@ -929,7 +964,7 @@ mainLoop:
 			case down:
 				historyAction = true
 				if historyStale {
-					historyPrefix = s.getHistoryByPrefix(string(line))
+					historyPrefix = s.getHistoryByPrefix(string(line), ModeGlobal)
 					historyPos = len(historyPrefix)
 					historyStale = false
 				}

main.go

@@ -66,6 +66,11 @@ func migrateDatabase(conn *sql.DB, currentVersion int) {
 	migrations := []string{
 		"ALTER TABLE history ADD COLUMN workdir varchar(4096) DEFAULT ''",
 		"ALTER TABLE history ADD COLUMN retval integer DEFAULT -9001",
+		"ALTER TABLE history ADD COLUMN unix_tmp integer",
+		"UPDATE history SET unix_tmp = strftime('%s', timestamp)",
+		"DROP VIEW count_by_date",
+		"ALTER TABLE history DROP COLUMN timestamp",
+		"ALTER TABLE history RENAME COLUMN unix_tmp TO timestamp",
 	}
 
 	if !(len(migrations) > currentVersion) {
@@ -157,13 +162,14 @@ type searchopts struct {
 	after   *time.Time
 	before  *time.Time
 	retval  *int
-	order   string
+	order   *string
+	limit   *int
 }
 
 func search(conn *sql.DB, opts searchopts) list.List {
 	args := make([]interface{}, 0)
 	var sb strings.Builder
-	sb.WriteString("SELECT id, command, workdir, user, hostname, retval ")
+	sb.WriteString("SELECT id, command, workdir, user, hostname, retval, timestamp ")
 	sb.WriteString("FROM history ")
 	sb.WriteString("WHERE 1=1 ") //1=1 so we can append as many AND foo as we want, or none
 
@@ -176,11 +182,11 @@ func search(conn *sql.DB, opts searchopts) list.List {
 		args = append(args, opts.workdir)
 	}
 	if opts.after != nil {
-		sb.WriteString("AND timestamp > datetime(?, 'unixepoch') ")
+		sb.WriteString("AND timestamp > ? ")
 		args = append(args, opts.after.Unix())
 	}
 	if opts.before != nil {
-		sb.WriteString("AND timestamp < datetime(?, 'unixepoch') ")
+		sb.WriteString("AND timestamp < ? ")
 		args = append(args, opts.before.Unix())
 	}
 	if opts.retval != nil {
@@ -188,7 +194,18 @@ func search(conn *sql.DB, opts searchopts) list.List {
 		args = append(args, opts.retval)
 	}
 	sb.WriteString("ORDER BY timestamp ")
-	sb.WriteString("ASC ")
+	if opts.order != nil {
+		sb.WriteString(*opts.order)
+		sb.WriteRune(' ')
+	} else {
+		sb.WriteString("ASC ")
+	}
+
+	if opts.limit != nil {
+		sb.WriteString("LIMIT ")
+		sb.WriteString(strconv.Itoa(*opts.limit))
+		sb.WriteRune(' ')
+	}
 
 	queryStmt := sb.String()
 
@@ -201,10 +218,12 @@ func search(conn *sql.DB, opts searchopts) list.List {
 	defer rows.Close()
 	for rows.Next() {
 		var entry HistoryEntry
-		err = rows.Scan(&entry.id, &entry.cmd, &entry.cwd, &entry.user, &entry.hostname, &entry.retval)
+		var timestamp int64
+		err = rows.Scan(&entry.id, &entry.cmd, &entry.cwd, &entry.user, &entry.hostname, &entry.retval, &timestamp)
 		if err != nil {
 			log.Panic(err)
 		}
+		entry.timestamp = time.Unix(timestamp, 0)
 		result.PushBack(&entry)
 	}
 	return result
@@ -220,7 +239,7 @@ func delete(conn *sql.DB, entryId uint32) {
 }
 
 func add(conn *sql.DB, entry HistoryEntry) {
-	stmt, err := conn.Prepare("INSERT INTO history (user, command, hostname, workdir, timestamp, retval) VALUES (?, ?, ?, ?, datetime(?, 'unixepoch'),?)")
+	stmt, err := conn.Prepare("INSERT INTO history (user, command, hostname, workdir, timestamp, retval) VALUES (?, ?, ?, ?, ?,?)")
 	if err != nil {
 		log.Panic(err)
 	}
@@ -309,6 +328,7 @@ func main() {
 				PROMPT_COMMAND='hs9001 add -ret $? "$(history 1)"'
 				bind -x '"\C-r": " READLINE_LINE=$(hs9001 bash-ctrlr 3>&1 1>&2 2>&3) READLINE_POINT=0"'
 			fi
+			alias hs='hs9001 search'
 		`)
 	case "bash-disable":
 		fmt.Printf("unset PROMPT_COMMAND\n")
@@ -338,13 +358,14 @@ func main() {
 		var afterTime string
 		var beforeTime string
 		var distinct bool = true
+		var today bool = false
 		var retVal int
 		searchCmd.StringVar(&workDir, "cwd", "", "Search only within this workdir")
 		searchCmd.StringVar(&afterTime, "after", "", "Start searching from this timeframe")
 		searchCmd.StringVar(&beforeTime, "before", "", "End searching from this timeframe")
 		searchCmd.BoolVar(&distinct, "distinct", true, "Remove consecutive duplicate commands from output")
+		searchCmd.BoolVar(&today, "today", false, "Search only today's entries. Overrides --after")
 		searchCmd.IntVar(&retVal, "ret", -9001, "Only query commands that returned with this exit code. -9001=all (default)")
-
 		searchCmd.Parse(globalargs)
 
 		args := searchCmd.Args()
@@ -352,7 +373,8 @@ func main() {
 		q := strings.Join(args, " ")
 
 		opts := searchopts{}
-		opts.order = "ASC"
+		o := "ASC"
+		opts.order = &o
 		if q != "" {
 			cmd := "%" + q + "%"
 			opts.command = &cmd
@@ -364,6 +386,11 @@ func main() {
 			}
 			opts.workdir = &wd
 		}
+
+		if today {
+			afterTime = "today"
+		}
+
 		if afterTime != "" {
 			afterTimestamp, err := naturaldate.Parse(afterTime, time.Now())
 			if err != nil {
@@ -384,13 +411,31 @@ func main() {
 		results := search(conn, opts)
 
 		previousCmd := ""
+
+		fi, err := os.Stdout.Stat()
+		if err != nil {
+			panic(err)
+		}
+
+		//Don't print colors if output is piped
+		printColors := true
+		if (fi.Mode() & os.ModeCharDevice) == 0 {
+			printColors = false
+		}
+
 		for e := results.Front(); e != nil; e = e.Next() {
 			entry, ok := e.Value.(*HistoryEntry)
 			if !ok {
 				log.Panic("Failed to retrieve entries")
 			}
 			if !distinct || previousCmd != entry.cmd {
-				fmt.Printf("%s\n", entry.cmd)
+				prefix := ""
+				postfix := ""
+				if printColors && entry.retval != 0 {
+					prefix = "\033[38;5;88m"
+					postfix = "\033[0m"
+				}
+				fmt.Printf("%s%s%s\n", prefix, entry.cmd, postfix)
 			}
 			previousCmd = entry.cmd
 		}