Skip to content

EOF Error When Inserting 1,000,000 Records with 1000 Concurrency Limit #700

@ermanimer

Description

@ermanimer

Hello,

While attempting to insert 1,000,000 records using a concurrency limit of 1000, I consistently receive between 100 to 200 EOF (End Of File) errors during each test run. The error occurs every time under these conditions, and I haven’t been able to determine the root cause despite multiple attempts to isolate the issue.

I also tried setting long timeout options to rule out any connection or latency-related problems, but that didn’t help either.

I’ve checked both the trace logs and the database logs, but couldn’t find any relevant information or errors that would explain this behavior. Interestingly, when using the godror library under the same conditions, the error does not occur.

Here is a simplified example of the code used:

package main

import (
	"context"
	"database/sql"
	"fmt"
	"io"
	"log/slog"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"

	_ "github.com/godror/godror"
	go_ora "github.com/sijms/go-ora/v2"
)

const (
	user     = ""
	password = ""
	server   = ""
	port     = 1521
	service  = ""
)

type connectFunc func(context.Context) (*sql.DB, error)

func main() {
	ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
	defer cancel()

	logger := newLogger()

	startedAt := time.Now()
	logger.Info("starting concurrent insertion test with go-ora", "start_time", startedAt)
	performConcurrentInsertionTest(ctx, logger, connectWithGOORA, 1000000, 1000)
	logger.Info("insertion test completed", "end_time", time.Now(), "duration", time.Since(startedAt))
}

func newLogger() *slog.Logger {
	logFile, err := os.OpenFile("benchmark.log", os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644)
	if err != nil {
		fmt.Fprintf(os.Stderr, "failed to open log file: %v\n", err)
		os.Exit(1)
	}

	logOutput := io.MultiWriter(os.Stdout, logFile)

	return slog.New(slog.NewTextHandler(logOutput, &slog.HandlerOptions{Level: slog.LevelDebug}))
}

func connectWithGOORA(ctx context.Context) (*sql.DB, error) {
	options := map[string]string{
		"TRACE FILE": "trace.log",
	}
	connString := go_ora.BuildUrl(server, port, service, user, password, options)
	db, err := sql.Open("oracle", connString)
	if err != nil {
		return nil, fmt.Errorf("failed to open database: %w", err)
	}

	if err := db.PingContext(ctx); err != nil {
		db.Close()
		return nil, fmt.Errorf("failed to ping database: %w", err)
	}

	return db, nil
}

func performConcurrentInsertionTest(ctx context.Context, logger *slog.Logger, connect connectFunc, recordCount int, concurrencyLimit int) {
	logger.Info("connecting")
	db, err := connect(ctx)
	if err != nil {
		logger.Error("failed to connect", "error", err)
		os.Exit(1)
	}
	logger.Info("connected")
	defer func() {
		logger.Info("closing connection")
		if err := db.Close(); err != nil {
			logger.Error("failed to close database", "error", err)
			os.Exit(1)
		}
		logger.Info("connection closed")
	}()

	logger.Info("creating table")
	if err := dropTable(ctx, db); err != nil {
		logger.Error("failed to drop table", "error", err)
		os.Exit(1)
	}
	if err := createTable(ctx, db); err != nil {
		logger.Error("failed to create table", "error", err)
		os.Exit(1)
	}
	logger.Info("table created")

	logger.Info("inserting records", "record_count", recordCount, "concurrency_limit", concurrencyLimit)
	semaphore := make(chan struct{}, concurrencyLimit)
	for i := range recordCount {
		if ctx.Err() != nil {
			return
		}

		semaphore <- struct{}{}
		go func() {
			defer func() {
				<-semaphore
			}()

			value := fmt.Sprintf("value %d", i)
			if err := insertRecord(ctx, db, value); err != nil {
				logger.Error("failed to insert record", "value", value, "error", err)
			}
		}()
	}

	for range concurrencyLimit {
		semaphore <- struct{}{}
	}
}

func dropTable(ctx context.Context, db *sql.DB) error {
	query := `
		DROP TABLE benchmark PURGE
	`
	_, err := db.ExecContext(ctx, query)
	if err != nil {
		if strings.Contains(err.Error(), "ORA-00942") {
			return nil // table doesn't exist
		}

		return fmt.Errorf("failed to execute query: %w", err)
	}

	return nil
}

func createTable(ctx context.Context, db *sql.DB) error {
	query := `
		CREATE TABLE benchmark (
			id NUMBER GENERATED AS IDENTITY PRIMARY KEY,
    		value VARCHAR2(255)
		)
	`
	_, err := db.ExecContext(ctx, query)
	if err != nil {
		return fmt.Errorf("failed to execute query: %w", err)
	}

	return nil
}

func insertRecord(ctx context.Context, db *sql.DB, value string) error {
	query := `
		INSERT INTO benchmark (value) VALUES (:1)
	`
	_, err := db.ExecContext(ctx, query, value)
	if err != nil {
		return fmt.Errorf("failed to execute query: %w", err)
	}

	return nil
}

Log Output:

time=2025-10-10T10:00:10.347+03:00 level=ERROR msg="failed to insert record" value="value 511" error="failed to execute query: EOF"

Go: v1.25.2
go-ora: v2.9.0
DB: Oracle Database 19c Enterprise Edition Release 19.0.0.0.0 - Production Version 19.26.0.0.0

trace.log

Metadata

Metadata

Assignees

No one assigned

    Labels

    No labels
    No labels

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions