Consistent socket hang up errors

[e10jon]

Unfortunately the library has become broken, as it seems that any large file will result in a "socket hang up" error. Observe an errored request below.

I think the solution involves setting Axios options around keep-alive. But there is no way to pass Axios options...

Error: socket hang up
<ref *2> Writable {
  _writableState: WritableState {
    objectMode: false,
    highWaterMark: 16384,
    finalCalled: false,
    needDrain: false,
    ending: false,
    ended: false,
    finished: false,
    destroyed: false,
    decodeStrings: true,
    defaultEncoding: 'utf8',
    length: 0,
    writing: false,
    corked: 0,
    sync: true,
    bufferProcessing: false,
    onwrite: [Function: bound onwrite],
    writecb: null,
    writelen: 0,
    afterWriteTickInfo: null,
    buffered: [],
    bufferedIndex: 0,
    allBuffers: true,
    allNoop: true,
    pendingcb: 0,
    constructed: true,
    prefinished: false,
    errorEmitted: false,
    emitClose: true,
    autoDestroy: true,
    errored: null,
    closed: false,
    closeEmitted: false,
    [Symbol(kOnFinished)]: []
  },
  _events: [Object: null prototype] {
    response: [Function: handleResponse],
    error: [Function: handleRequestError]
  },
  _eventsCount: 2,
  _maxListeners: undefined,
  _options: {
    maxRedirects: 21,
    maxBodyLength: 10485760,
    protocol: 'https:',
    path: '/convert/docx/to/jpg?secret=XXX',
    method: 'POST',
    headers: {
      Accept: 'application/json',
      'Content-Type': 'application/x-www-form-urlencoded',
      'User-Agent': 'ConvertAPI-Node/1.11.2',
      'Content-Length': 52
    },
    agent: undefined,
    agents: { http: undefined, https: undefined },
    auth: undefined,
    hostname: 'v2.convertapi.com',
    port: null,
    nativeProtocols: { 'http:': [Object], 'https:': [Object] },
    pathname: '/convert/docx/to/jpg',
    search: '?secret=XXX'
  },
  _ended: true,
  _ending: true,
  _redirectCount: 0,
  _redirects: [],
  _requestBodyLength: 52,
  _requestBodyBuffers: [
    {
      data: <Buffer 46 69 6c 65 3d 77 75 65 32 79 6c 31 63 6c 30 6f 74 67 64 71 35 70 33 62 71 38 38 30 37 77 31 38 73 70 34 35 77 26 53 74 6f 72 65 46 69 6c 65 3d 74 72 ... 2 more bytes>,
      encoding: undefined
    }
  ],
  _onNativeResponse: [Function (anonymous)],
  _currentRequest: <ref *1> ClientRequest {
    _events: [Object: null prototype] {
      response: [Function],
      abort: [Function (anonymous)],
      aborted: [Function (anonymous)],
      connect: [Function (anonymous)],
      error: [Function (anonymous)],
      socket: [Function (anonymous)],
      timeout: [Function (anonymous)]
    },
    _eventsCount: 7,
    _maxListeners: undefined,
    outputData: [],
    outputSize: 0,
    writable: true,
    destroyed: true,
    _last: false,
    chunkedEncoding: false,
    shouldKeepAlive: true,
    maxRequestsOnConnectionReached: false,
    _defaultKeepAlive: true,
    useChunkedEncodingByDefault: true,
    sendDate: false,
    _removedConnection: false,
    _removedContLen: false,
    _removedTE: false,
    strictContentLength: false,
    _contentLength: 52,
    _hasBody: true,
    _trailer: '',
    finished: true,
    _headerSent: true,
    _closed: true,
    socket: TLSSocket {
      _tlsOptions: [Object],
      _secureEstablished: true,
      _securePending: false,
      _newSessionPending: false,
      _controlReleased: true,
      secureConnecting: false,
      _SNICallback: null,
      servername: 'v2.convertapi.com',
      alpnProtocol: false,
      authorized: true,
      authorizationError: null,
      encrypted: true,
      _events: [Object: null prototype],
      _eventsCount: 11,
      connecting: false,
      _hadError: true,
      _parent: null,
      _host: 'v2.convertapi.com',
      _closeAfterHandlingError: false,
      _readableState: [ReadableState],
      _maxListeners: undefined,
      _writableState: [WritableState],
      allowHalfOpen: false,
      _sockname: null,
      _pendingData: null,
      _pendingEncoding: '',
      server: undefined,
      _server: null,
      ssl: null,
      _requestCert: true,
      _rejectUnauthorized: true,
      timeout: 5000,
      parser: null,
      _httpMessage: [Circular *1],
      [Symbol(alpncallback)]: null,
      [Symbol(res)]: null,
      [Symbol(verified)]: true,
      [Symbol(pendingSession)]: null,
      [Symbol(async_id_symbol)]: 165,
      [Symbol(kHandle)]: null,
      [Symbol(lastWriteQueueSize)]: 0,
      [Symbol(timeout)]: Timeout {
        _idleTimeout: -1,
        _idlePrev: null,
        _idleNext: null,
        _idleStart: 3593,
        _onTimeout: null,
        _timerArgs: undefined,
        _repeat: null,
        _destroyed: true,
        [Symbol(refed)]: false,
        [Symbol(kHasPrimitive)]: false,
        [Symbol(asyncId)]: 163,
        [Symbol(triggerId)]: 161
      },
      [Symbol(kBuffer)]: null,
      [Symbol(kBufferCb)]: null,
      [Symbol(kBufferGen)]: null,
      [Symbol(kCapture)]: false,
      [Symbol(kSetNoDelay)]: false,
      [Symbol(kSetKeepAlive)]: true,
      [Symbol(kSetKeepAliveInitialDelay)]: 1,
      [Symbol(kBytesRead)]: 537,
      [Symbol(kBytesWritten)]: 96365,
      [Symbol(connect-options)]: [Object]
    },
    _header: 'POST /convert/docx/to/jpg?secret=XXX HTTP/1.1\r\n' +
      'Accept: application/json\r\n' +
      'Content-Type: application/x-www-form-urlencoded\r\n' +
      'User-Agent: ConvertAPI-Node/1.11.2\r\n' +
      'Content-Length: 52\r\n' +
      'Host: v2.convertapi.com\r\n' +
      'Connection: keep-alive\r\n' +
      '\r\n',
    _keepAliveTimeout: 0,
    _onPendingData: [Function: nop],
    agent: Agent {
      _events: [Object: null prototype],
      _eventsCount: 2,
      _maxListeners: undefined,
      defaultPort: 443,
      protocol: 'https:',
      options: [Object: null prototype],
      requests: [Object: null prototype] {},
      sockets: [Object: null prototype] {},
      freeSockets: [Object: null prototype] {},
      keepAliveMsecs: 1000,
      keepAlive: true,
      maxSockets: Infinity,
      maxFreeSockets: 256,
      scheduling: 'lifo',
      maxTotalSockets: Infinity,
      totalSocketCount: 0,
      maxCachedSessions: 100,
      _sessionCache: [Object],
      [Symbol(kCapture)]: false
    },
    socketPath: undefined,
    method: 'POST',
    maxHeaderSize: undefined,
    insecureHTTPParser: undefined,
    joinDuplicateHeaders: undefined,
    path: '/convert/docx/to/jpg?secret=XXX',
    _ended: false,
    res: null,
    aborted: false,
    timeoutCb: [Function: emitRequestTimeout],
    upgradeOrConnect: false,
    parser: null,
    maxHeadersCount: null,
    reusedSocket: true,
    host: 'v2.convertapi.com',
    protocol: 'https:',
    _redirectable: [Circular *2],
    [Symbol(kCapture)]: false,
    [Symbol(kBytesWritten)]: 0,
    [Symbol(kNeedDrain)]: false,
    [Symbol(corked)]: 0,
    [Symbol(kOutHeaders)]: [Object: null prototype] {
      accept: [Array],
      'content-type': [Array],
      'user-agent': [Array],
      'content-length': [Array],
      host: [Array]
    },
    [Symbol(errored)]: null,
    [Symbol(kHighWaterMark)]: 16384,
    [Symbol(kRejectNonStandardBodyWrites)]: false,
    [Symbol(kUniqueHeaders)]: null
  },
  _currentUrl: 'https://v2.convertapi.com/convert/docx/to/jpg?secret=XXX',
  [Symbol(kCapture)]: false
}

[Irto]

I have been through the same issue at the same endpoint. 😞 Someone knows any workaround?

[laurynas-convertapi]

Hi @e10jon, @Irto I'd like to understand better what changed:

1. Have you been using same version of the library before with the same size files successfully?
2. Did you upgrade axios and the failures started after upgrade?
3. Does it fail only with big files? Maybe you noticed approximately from which size the failures start?

Better understanding of the causes will help us with the appropriate fix.

[Irto]

Hey @laurynas-baltsoft !

Have you been using same version of the library before with the same size files successfully?

I'm using convertapi @ 1.12.0.

Did you upgrade axios and the failures started after upgrade?

I'm a new user, I've integrated the convertapi at the end of last week. Never did a upgrade.

Does it fail only with big files? Maybe you noticed approximately from which size the failures start?

TBH I'm not sure it's related to the size of file. I've got some issue on files with less than 100KB. I've workaround this issue by adding a retry on may async job. So, I guess is probably some intermittent issue on your server side because next try it works.

[e10jon]

Thanks for responding! Unfortunately my usage of your API is still in a pre-production phase, so I haven’t been able to observe if file size is a factor. What I know is that when I first started using your API (maybe 1-2 months ago), the results were fantastic. Then, at some point, the results degraded, and now it seems like only 25% of API calls are successful. Sent from a cellular telephone

…

On Mon, Sep 25, 2023 at 12:11 PM Israel Trindade ***@***.***> wrote: Hey @laurynas-baltsoft <https://github.com/laurynas-baltsoft> ! Have you been using same version of the library before with the same size files successfully? I'm using convertapi @ 1.12.0. Did you upgrade axios and the failures started after upgrade? I'm a new user, I've integrated the convertapi at the end of last week. Never did a upgrade. Does it fail only with big files? Maybe you noticed approximately from which size the failures start? TBH I'm not sure it's related to the size of file. I've got some issue on files with less than 100KB. I've workaround this issue by adding a retry on may async job. So, I guess is probably some intermittent issue on your server side because next try it works. — Reply to this email directly, view it on GitHub <#68 (comment)>, or unsubscribe <https://github.com/notifications/unsubscribe-auth/AAATT4I37VYOQOTVHVVPQSTX4HJMVANCNFSM6AAAAAA46J7PWY> . You are receiving this because you were mentioned.Message ID: ***@***.***>

[tomasr78]

Hello everyone, would you kindly share your integration codes? Also, I am curious to know if you use Async in your process.

[e10jon]

Sure. My integration is very simply. It looks like this:

const convertApi = new ConvertApi('XXX')
const convertRes = await convertApi.convert('jpg', { File: 'path/to/file.docx' })

[tomasr78]

@e10jon could you confirm that your files convert fine on the live demo page? https://www.convertapi.com/pdf-to-jpg

[e10jon]

I went ahead and created my own simple client library. It works every time I make an API call now. Feel free to reuse as you please!

import { basename, extname } from 'node:path'

import { readFile, writeFile } from 'fs/promises'

export class ConvertApi {
  constructor(private secret: string) {}

  async convert(format: 'png' | 'jpg', pathToFile: string) {
    const file = await readFile(pathToFile)
    const base64File = file.toString('base64')
    const filename = basename(pathToFile)
    const extension = extname(pathToFile).replace(/\./, '')

    const req = await fetch(`https://v2.convertapi.com/convert/${extension}/to/${format}?Secret=${this.secret}`, {
      method: 'POST',
      body: JSON.stringify({
        Parameters: [
          {
            Name: 'File',
            FileValue: {
              Name: filename,
              Data: base64File,
            },
          },
          {
            Name: 'StoreFile',
            Value: false,
          },
        ],
      }),
      headers: {
        'Content-Type': 'application/json',
      },
    })
    const res = (await req.json()) as { Files: { FileName: string; FileExt: string; FileSize: number; FileData: string }[] }

    return {
      files: res.Files.map((data) => ({
        fileName: data.FileName,
        save: async (path: string) => {
          await writeFile(path, Buffer.from(data.FileData, 'base64'))
        },
      })),
    }
  }
}

[tomasr78]

It appears that you are utilizing JSON to post a file to the API, but this method may not be the best approach. Instead, it is recommended to use either multipart/form-data or application/octet-stream as the content type for posting the file, rather than application/json.

Using application/json has its drawbacks, including:

• The need to encode the file to Base64, which increases the payload by 30%
• Limitations on the size of JSON files that can be posted, which can prevent uploading larger files.

[laurynas-convertapi]

Thanks for the input! Keep alive might help in this situation. Adding an option to the client (enabled by default):
#69

[laurynas-convertapi]

Enabled keep-alive in new v1.13.0 version.