From 6ad59c9990889cb98de6225c3b8125c7c7bb1c1a Mon Sep 17 00:00:00 2001
From: PTah <papatramp@gmail.com>
Date: Mon, 13 Apr 2026 15:33:25 +1000
Subject: [PATCH] =?UTF-8?q?fix:=20=D0=BE=D0=B3=D1=80=D0=B0=D0=BD=D0=B8?=
 =?UTF-8?q?=D1=87=D0=B8=D1=82=D1=8C=20=D0=BF=D1=83=D0=BB=20Postgres=20?=
 =?UTF-8?q?=D0=B8=20=D1=80=D0=B5=D0=B6=D0=B5=20=D0=BF=D0=B8=D1=81=D0=B0?=
 =?UTF-8?q?=D1=82=D1=8C=20=D0=BF=D1=80=D0=BE=D0=B3=D1=80=D0=B5=D1=81=D1=81?=
 =?UTF-8?q?=20=D1=81=D0=BA=D0=B0=D0=BD=D0=B0?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- DB_MAX_OPEN_CONNS (25), DB_MAX_IDLE_CONNS (10), DB_CONN_MAX_LIFETIME_SEC (1800)

- Throttle UpdateScan прогресса в Runner (500ms), меньше одновременных запросов к БД

Устраняет FATAL too many clients при больших сканах и частом опросе GetScan

Made-with: Cursor
---
 cmd/server/main.go        |  9 +++++++
 internal/config/config.go | 51 +++++++++++++++++++++++++++++++++------
 internal/scans/runner.go  | 23 +++++++++++++++---
 3 files changed, 73 insertions(+), 10 deletions(-)

diff --git a/cmd/server/main.go b/cmd/server/main.go
index c7a4513..e7f673b 100644
--- a/cmd/server/main.go
+++ b/cmd/server/main.go
@@ -57,6 +57,15 @@ func makeStore(cfg config.Config) (scans.Store, func()) {
 		log.Fatalf("db ping failed: %v", err)
 	}
 
+	db.SetMaxOpenConns(cfg.DBMaxOpenConns)
+	db.SetMaxIdleConns(cfg.DBMaxIdleConns)
+	if cfg.DBConnMaxLifetime > 0 {
+		db.SetConnMaxLifetime(cfg.DBConnMaxLifetime)
+		log.Printf("postgres pool: max_open=%d max_idle=%d conn_max_lifetime=%v", cfg.DBMaxOpenConns, cfg.DBMaxIdleConns, cfg.DBConnMaxLifetime)
+	} else {
+		log.Printf("postgres pool: max_open=%d max_idle=%d conn_max_lifetime=off", cfg.DBMaxOpenConns, cfg.DBMaxIdleConns)
+	}
+
 	if cfg.RunMigrations {
 		if err = store.RunMigrations(db); err != nil {
 			_ = db.Close()
diff --git a/internal/config/config.go b/internal/config/config.go
index 06f0c3e..cd3cb57 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -2,7 +2,9 @@ package config
 
 import (
 	"os"
+	"strconv"
 	"strings"
+	"time"
 )
 
 type Config struct {
@@ -13,6 +15,10 @@ type Config struct {
 	SNMPEnabled     bool
 	SNMPCommunity   string
 	ResetDBSecret   string // NETTOPO_RESET_DB_SECRET — секрет для POST /api/admin/purge-scans
+	// Пул соединений к PostgreSQL (см. database/sql). По умолчанию ограничен, иначе при большом скане легко исчерпать max_connections на сервере.
+	DBMaxOpenConns    int
+	DBMaxIdleConns    int
+	DBConnMaxLifetime time.Duration
 }
 
 func FromEnv() Config {
@@ -44,13 +50,44 @@ func FromEnv() Config {
 		snmpCommunity = "public"
 	}
 
+	maxOpen := intFromEnvMin("DB_MAX_OPEN_CONNS", 25, 1)
+	maxIdle := intFromEnvMin("DB_MAX_IDLE_CONNS", 10, 0)
+	if maxIdle > maxOpen {
+		maxIdle = maxOpen
+	}
+	connLife := 30 * time.Minute
+	if s := strings.TrimSpace(os.Getenv("DB_CONN_MAX_LIFETIME_SEC")); s != "" {
+		if sec, err := strconv.Atoi(s); err == nil {
+			if sec == 0 {
+				connLife = 0 // без ограничения времени жизни соединения
+			} else if sec > 0 {
+				connLife = time.Duration(sec) * time.Second
+			}
+		}
+	}
+
 	return Config{
-		HTTPAddr:      addr,
-		DBDSN:         dbDSN,
-		StoreBackend:  storeBackend,
-		RunMigrations: runMigrations,
-		SNMPEnabled:   snmpEnabled,
-		SNMPCommunity: snmpCommunity,
-		ResetDBSecret: strings.TrimSpace(os.Getenv("NETTOPO_RESET_DB_SECRET")),
+		HTTPAddr:          addr,
+		DBDSN:             dbDSN,
+		StoreBackend:      storeBackend,
+		RunMigrations:     runMigrations,
+		SNMPEnabled:       snmpEnabled,
+		SNMPCommunity:     snmpCommunity,
+		ResetDBSecret:     strings.TrimSpace(os.Getenv("NETTOPO_RESET_DB_SECRET")),
+		DBMaxOpenConns:    maxOpen,
+		DBMaxIdleConns:    maxIdle,
+		DBConnMaxLifetime: connLife,
 	}
 }
+
+func intFromEnvMin(key string, def, min int) int {
+	s := strings.TrimSpace(os.Getenv(key))
+	if s == "" {
+		return def
+	}
+	n, err := strconv.Atoi(s)
+	if err != nil || n < min {
+		return def
+	}
+	return n
+}
diff --git a/internal/scans/runner.go b/internal/scans/runner.go
index f3db7ab..d7d1c88 100644
--- a/internal/scans/runner.go
+++ b/internal/scans/runner.go
@@ -163,11 +163,28 @@ func (r *Runner) run(ctx context.Context, job ScanJob) {
 	workCh := make(chan string)
 	wg := sync.WaitGroup{}
 
-	pushProgress := func() {
+	// Каждый UpdateScan в Postgres тянет GetScan+UPDATE; при сотнях воркеров без паузы исчерпывается пул соединений к БД.
+	const minProgressWriteInterval = 500 * time.Millisecond
+	var progWriteMu sync.Mutex
+	var lastProgressWrite time.Time
+
+	pushProgress := func(force bool) {
 		mu.Lock()
 		q, f, u := quickDone, fullDone, up
 		mu.Unlock()
 		prog := scanProgressBlend(q, f, total)
+
+		if !force {
+			progWriteMu.Lock()
+			now := time.Now()
+			if !lastProgressWrite.IsZero() && now.Sub(lastProgressWrite) < minProgressWriteInterval {
+				progWriteMu.Unlock()
+				return
+			}
+			lastProgressWrite = now
+			progWriteMu.Unlock()
+		}
+
 		_, _ = r.store.UpdateScan(job.ID, ScanUpdate{
 			Status:   "running",
 			Progress: prog,
@@ -207,7 +224,7 @@ func (r *Runner) run(ctx context.Context, job ScanJob) {
 			}
 			quickDone++
 			mu.Unlock()
-			pushProgress()
+			pushProgress(false)
 
 			if isUp && r.cfg.SNMPEnabled {
 				snmpRes, lldpRes, ifRes, portDevRes := probeSNMPAndLLDP(ip, r.cfg.SNMPCommunity, checkedAt, job.Options.PingTimeoutMS)
@@ -232,7 +249,7 @@ func (r *Runner) run(ctx context.Context, job ScanJob) {
 			mu.Lock()
 			fullDone++
 			mu.Unlock()
-			pushProgress()
+			pushProgress(false)
 		}
 	}