Skip to content

Commit

Permalink
minor fixups
Browse files Browse the repository at this point in the history
Signed-off-by: Ettore Di Giacinto <[email protected]>
  • Loading branch information
mudler committed Oct 11, 2024
1 parent 53cff5c commit 8055c87
Showing 1 changed file with 9 additions and 2 deletions.
11 changes: 9 additions & 2 deletions pkg/model/initializers.go
Original file line number Diff line number Diff line change
Expand Up @@ -449,13 +449,20 @@ func (ml *ModelLoader) BackendLoader(opts ...Option) (client grpc.Backend, err e
}

model, err := ml.LoadModel(o.modelID, o.model, ml.grpcModel(backendToConsume, AutoDetect, o))
if backend == LLamaCPP && err != nil {
if err != nil {
// XXX: This is too backend specific(llama-cpp), remove this bit or generalize further
// We failed somehow starting the binary. For instance, could be that we are missing
// some libraries if running in binary-only mode.
// In this case, we attempt to load the model with the fallback variant.

// If not llama-cpp backend, return error immediately
if backend != LLamaCPP {
return nil, err
}

// Otherwise attempt with fallback
log.Error().Msgf("[%s] Failed loading model, trying with fallback '%s'", backend, LLamaCPPFallback)
model, err = ml.LoadModel(o.modelID, o.model, ml.grpcModel(LLamaCPPFallback, AutoDetect, o))
model, err = ml.LoadModel(o.modelID, o.model, ml.grpcModel(LLamaCPPFallback, false, o))
if err != nil {
return nil, err
}
Expand Down

0 comments on commit 8055c87

Please sign in to comment.