Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion biome.jsonc
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@
"noAssignInExpressions": "error",
"noAsyncPromiseExecutor": "error",
"noDoubleEquals": "error",
"noExplicitAny": "warn",
"noExplicitAny": "error",
"noFocusedTests": "error",
"noImplicitAnyLet": "error",
"noShadowRestrictedNames": "error",
Expand Down
2 changes: 1 addition & 1 deletion src/http/error-handler.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ export const setErrorHandler = (
app: FastifyInstance,
options?: {
respectStatusCode?: boolean
formatter?: (error: StorageError) => Record<string, any>
formatter?: (error: StorageError) => unknown
}
) => {
app.setErrorHandler<Error>(function (error, request, reply) {
Expand Down
23 changes: 17 additions & 6 deletions src/http/plugins/log-request.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { logger, logSchema, redactQueryParamFromRequest } from '@internal/monitoring'
import { RouteGenericInterface } from 'fastify'
import { FastifyReply } from 'fastify/types/reply'
import { FastifyRequest } from 'fastify/types/request'
import fastifyPlugin from 'fastify-plugin'
Expand All @@ -7,6 +8,11 @@ interface RequestLoggerOptions {
excludeUrls?: string[]
}

type RawRequestMetadata = FastifyRequest['raw'] & {
executionError?: Error
resources?: string[]
}

declare module 'fastify' {
interface FastifyRequest {
executionError?: Error
Expand All @@ -18,8 +24,8 @@ declare module 'fastify' {

interface FastifyContextConfig {
operation?: { type: string }
resources?: (req: FastifyRequest<any>) => string[]
logMetadata?: (req: FastifyRequest<any>) => Record<string, unknown>
resources?(req: FastifyRequest<RouteGenericInterface>): string[]
logMetadata?(req: FastifyRequest<RouteGenericInterface>): Record<string, unknown>
}
}

Expand Down Expand Up @@ -58,11 +64,12 @@ export const logRequest = (options: RequestLoggerOptions) =>
*/
fastify.addHook('preHandler', async (req) => {
const resourceFromParams = Object.values(req.params || {}).join('/')
const rawReq = getRawRequest(req)
const resources = getFirstDefined<string[]>(
req.resources,
req.routeOptions.config.resources?.(req),
(req.raw as any).resources,
resourceFromParams ? [resourceFromParams] : ([] as string[])
rawReq.resources,
resourceFromParams ? [resourceFromParams] : []
)

if (resources && resources.length > 0) {
Expand Down Expand Up @@ -123,7 +130,7 @@ function doRequestLog(req: FastifyRequest, options: LogRequestOptions) {
const rId = req.id
const cIP = req.ip
const statusCode = options.statusCode
const error = (req.raw as any).executionError || req.executionError
const error = getRawRequest(req).executionError || req.executionError
const tenantId = req.tenantId

let reqMetadata: Record<string, unknown> = {}
Expand Down Expand Up @@ -171,7 +178,11 @@ function doRequestLog(req: FastifyRequest, options: LogRequestOptions) {
})
}

function getFirstDefined<T>(...values: any[]): T | undefined {
function getRawRequest(req: FastifyRequest): RawRequestMetadata {
return req.raw as RawRequestMetadata
}

function getFirstDefined<T>(...values: (T | undefined)[]): T | undefined {
for (const value of values) {
if (value !== undefined) {
return value
Expand Down
6 changes: 5 additions & 1 deletion src/http/routes-helper.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
type BucketResponseType = { message: string; statusCode?: string; error?: string }
type SchemaObject = Record<string, unknown>

/**
* Create generic response for all buckets
Expand All @@ -23,7 +24,10 @@ function createResponse(message: string, status?: string, error?: string): Bucke
return response
}

function createDefaultSchema(successResponseSchema: any, properties: any): any {
function createDefaultSchema(
successResponseSchema: SchemaObject,
properties: SchemaObject
): SchemaObject {
return {
headers: { $ref: 'authSchema#' },
response: {
Expand Down
14 changes: 12 additions & 2 deletions src/http/routes/admin/migrations.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,15 @@ import apiKey from '../../plugins/apikey'

const { pgQueueEnable } = getConfig()

type ResetFleetBody = {
untilMigration?: unknown
markCompletedTillMigration?: unknown
}

type FailedQuery = {
cursor?: string
}

export default async function routes(fastify: FastifyInstance) {
fastify.register(apiKey)

Expand All @@ -30,7 +39,7 @@ export default async function routes(fastify: FastifyInstance) {
return reply.status(400).send({ message: 'Queue is not enabled' })
}

const { untilMigration, markCompletedTillMigration } = req.body as Record<string, unknown>
const { untilMigration, markCompletedTillMigration } = req.body as ResetFleetBody

if (!isDBMigrationName(untilMigration)) {
return reply.status(400).send({ message: 'Invalid migration' })
Expand Down Expand Up @@ -95,7 +104,8 @@ export default async function routes(fastify: FastifyInstance) {
if (!pgQueueEnable) {
return reply.code(400).send({ message: 'Queue is not enabled' })
}
const offset = (req.query as any).cursor ? Number((req.query as any).cursor) : 0
const { cursor } = req.query as FailedQuery
const offset = cursor ? Number(cursor) : 0

const failed = await multitenantKnex
.table('tenants')
Expand Down
14 changes: 9 additions & 5 deletions src/http/routes/iceberg/table.ts
Original file line number Diff line number Diff line change
Expand Up @@ -518,12 +518,10 @@ export default async function routes(fastify: FastifyInstance) {
try {
if (typeof payload === 'string') return done(null, JSONBigint.parse(payload))
if (Buffer.isBuffer(payload)) return done(null, JSONBigint.parse(payload.toString('utf8')))
if (payload && typeof (payload as any).on === 'function') {
if (isReadablePayload(payload)) {
const chunks: Buffer[] = []
;(payload as NodeJS.ReadableStream).on('data', (c) =>
chunks.push(Buffer.isBuffer(c) ? c : Buffer.from(String(c)))
)
;(payload as NodeJS.ReadableStream).on('end', () => {
payload.on('data', (c) => chunks.push(Buffer.isBuffer(c) ? c : Buffer.from(String(c))))
payload.on('end', () => {
try {
done(null, JSONBigint.parse(Buffer.concat(chunks).toString('utf8')))
} catch (err) {
Expand Down Expand Up @@ -574,3 +572,9 @@ export default async function routes(fastify: FastifyInstance) {
)
})
}

function isReadablePayload(payload: unknown): payload is NodeJS.ReadableStream {
return (
!!payload && typeof payload === 'object' && 'on' in payload && typeof payload.on === 'function'
)
}
2 changes: 1 addition & 1 deletion src/http/routes/render/rate-limiter.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ const {
rateLimiterRenderPathMaxReqSec,
} = getConfig()

export const rateLimiter = fp((fastify: FastifyInstance, ops: any, done: () => void) => {
export const rateLimiter = fp((fastify: FastifyInstance, _ops: unknown, done: () => void) => {
fastify.register(fastifyRateLimit, {
global: true,
max: rateLimiterRenderPathMaxReqSec * 4,
Expand Down
25 changes: 22 additions & 3 deletions src/http/routes/s3/error-handler.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,18 @@ import { FastifyReply } from 'fastify/types/reply'
import { FastifyRequest } from 'fastify/types/request'
import { DatabaseError } from 'pg'

type ValidationIssue = {
instancePath?: string
message?: string
}

export const s3ErrorHandler = (
error: FastifyError | Error,
request: FastifyRequest,
reply: FastifyReply
) => {
request.executionError = error
const validation = getValidationIssues(error)

const resource = request.url
.split('?')[0]
Expand All @@ -19,12 +25,12 @@ export const s3ErrorHandler = (
.filter((e) => e)
.join('/')

if ('validation' in error) {
if (validation) {
return reply.status(400).send({
Error: {
Resource: resource,
Code: ErrorCode.InvalidRequest,
Message: formatValidationError(error.validation).message,
Message: formatValidationError(validation).message,
},
})
}
Expand Down Expand Up @@ -78,7 +84,20 @@ export const s3ErrorHandler = (
})
}

function formatValidationError(errors: any) {
function isValidationIssueArray(value: unknown): value is ValidationIssue[] {
return Array.isArray(value)
}

function getValidationIssues(error: FastifyError | Error): ValidationIssue[] | undefined {
if (!('validation' in error)) {
return undefined
}

const value = error.validation
return isValidationIssueArray(value) ? value : undefined
}

function formatValidationError(errors: readonly ValidationIssue[]) {
let text = ''
const separator = ', '

Expand Down
4 changes: 2 additions & 2 deletions src/http/routes/s3/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -66,12 +66,12 @@ export default async function routes(fastify: FastifyInstance) {
req.opentelemetry()?.span?.setAttribute('http.operation', req.operation.type)
}

const data: RequestInput<any> = {
const data = {
Params: req.params,
Body: req.body,
Headers: req.headers,
Querystring: req.query,
}
} as unknown as RequestInput<typeof route.schema>
const compiler = route.compiledSchema()
const isValid = compiler(data)

Expand Down
69 changes: 52 additions & 17 deletions src/http/routes/s3/router.ts
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,7 @@ type Route<S extends Schema, Context> = {
compiledSchema: () => ValidateFunction<JTDDataType<S>>
}

interface RouteOptions<S extends JSONSchema> {
interface RouteOptions<S extends Schema> {
disableContentTypeParser?: boolean
acceptMultiformData?: boolean
operation: string
Expand All @@ -134,11 +134,11 @@ export class Router<Context = unknown, S extends Schema = Schema> {
allErrors: false,
})

registerRoute<R extends S = S>(
registerRoute(
method: HTTPMethod,
url: string,
options: RouteOptions<R>,
handler: Handler<R, Context>
options: RouteOptions<Schema>,
handler: Handler<Schema, Context>
) {
const { query, headers } = this.parseRequestInfo(url)
const normalizedUrl = url.split('?')[0].split('|')[0]
Expand Down Expand Up @@ -194,14 +194,15 @@ export class Router<Context = unknown, S extends Schema = Schema> {
)
}

const newRoute: Route<R, Context> = {
const newRoute: Route<Schema, Context> = {
method: method as HTTPMethod,
path: normalizedUrl,
querystringMatches: query,
headersMatches: headers,
schema,
compiledSchema: () => this.ajv.getSchema(method + url) as ValidateFunction<JTDDataType<R>>,
handler: handler as Handler<R, Context>,
compiledSchema: () =>
this.ajv.getSchema(method + url) as ValidateFunction<JTDDataType<Schema>>,
handler,
disableContentTypeParser,
acceptMultiformData,
operation,
Expand All @@ -217,24 +218,58 @@ export class Router<Context = unknown, S extends Schema = Schema> {
this._routes.set(normalizedUrl, existingPath)
}

get<R extends S>(url: string, options: RouteOptions<R>, handler: Handler<R, Context>) {
this.registerRoute('get', url, options, handler as any)
private registerRouteUnsafe(
method: HTTPMethod,
url: string,
options: RouteOptions<Schema>,
handler: Handler<Schema, Context>
) {
this.registerRoute(method, url, options, handler)
}

get<R extends Schema>(url: string, options: RouteOptions<R>, handler: Handler<R, Context>) {
this.registerRouteUnsafe(
'get',
url,
options as unknown as RouteOptions<Schema>,
handler as unknown as Handler<Schema, Context>
)
}

post<R extends S = S>(url: string, options: RouteOptions<R>, handler: Handler<R, Context>) {
this.registerRoute('post', url, options, handler as any)
post<R extends Schema>(url: string, options: RouteOptions<R>, handler: Handler<R, Context>) {
this.registerRouteUnsafe(
'post',
url,
options as unknown as RouteOptions<Schema>,
handler as unknown as Handler<Schema, Context>
)
}

put<R extends S = S>(url: string, options: RouteOptions<R>, handler: Handler<R, Context>) {
this.registerRoute('put', url, options, handler as any)
put<R extends Schema>(url: string, options: RouteOptions<R>, handler: Handler<R, Context>) {
this.registerRouteUnsafe(
'put',
url,
options as unknown as RouteOptions<Schema>,
handler as unknown as Handler<Schema, Context>
)
}

delete<R extends S = S>(url: string, options: RouteOptions<R>, handler: Handler<R, Context>) {
this.registerRoute('delete', url, options, handler as any)
delete<R extends Schema>(url: string, options: RouteOptions<R>, handler: Handler<R, Context>) {
this.registerRouteUnsafe(
'delete',
url,
options as unknown as RouteOptions<Schema>,
handler as unknown as Handler<Schema, Context>
)
}

head<R extends S = S>(url: string, options: RouteOptions<R>, handler: Handler<R, Context>) {
this.registerRoute('head', url, options, handler as any)
head<R extends Schema>(url: string, options: RouteOptions<R>, handler: Handler<R, Context>) {
this.registerRouteUnsafe(
'head',
url,
options as unknown as RouteOptions<Schema>,
handler as unknown as Handler<Schema, Context>
)
}

parseQueryMatch(query: string) {
Expand Down
10 changes: 6 additions & 4 deletions src/http/routes/tus/lifecycle.ts
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ function getNodeRequest(rawReq: Request): MultiPartRequest {
return req
}
export type MultiPartRequest = http.IncomingMessage & {
executionError?: Error
log: BaseLogger
upload: {
storage: Storage
Expand Down Expand Up @@ -282,8 +283,9 @@ export async function onUploadFinish(rawReq: Request, upload: Upload) {
}
} catch (e) {
if (isRenderableError(e)) {
;(e as any).status_code = parseInt(e.render().statusCode, 10)
;(e as any).body = e.render().message
const renderableError = e as unknown as Error & Partial<TusError>
renderableError.status_code = parseInt(e.render().statusCode, 10)
renderableError.body = e.render().message
}
throw e
}
Expand All @@ -298,9 +300,9 @@ export function onResponseError(rawReq: Request, e: TusError | Error) {
const req = getNodeRequest(rawReq)

if (e instanceof Error) {
;(req as any).executionError = e
req.executionError = e
} else {
;(req as any).executionError = ERRORS.TusError(e.body, e.status_code).withMetadata(e)
req.executionError = ERRORS.TusError(e.body, e.status_code).withMetadata(e)
}

if (isRenderableError(e)) {
Expand Down
2 changes: 1 addition & 1 deletion src/http/routes/vector/query-vectors.ts
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ export default async function routes(fastify: FastifyInstance) {
coerceTypes: false,
})

const perRouteValidator: FastifySchemaCompiler<any> = ({ schema }) => {
const perRouteValidator: FastifySchemaCompiler<unknown> = ({ schema }) => {
const validate = ajvNoRemoval.compile(schema as object)
return (data) => {
const ok = validate(data)
Expand Down
Loading