Issue/1843 make WebID profile fetch respect NODE_TLS_REJECT_UNAUTHORIZED=0

lib/handlers/get.mjs

@@ -77,13 +77,16 @@ export default async function handler (req, res, next) {
   let contentRange
   let chunksize
 
-  if (ret) {
-    stream = ret.stream
-    contentType = ret.contentType
-    container = ret.container
-    contentRange = ret.contentRange
-    chunksize = ret.chunksize
-  }
+  if (ret) {
+    stream = ret.stream
+    contentType = ret.contentType
+    container = ret.container
+    contentRange = ret.contentRange
+    chunksize = ret.chunksize
+    if (ret.modified) {
+      res.header('Last-Modified', ret.modified.toUTCString())
+    }
+  }
 
   // Till here it must exist
   if (!includeBody) {

lib/ldp.mjs

@@ -451,7 +451,7 @@ class LDP {
     }
 
     if (!options.includeBody) {
-      return { stream: stats, contentType, container: stats.isDirectory() }
+      return { stream: stats, contentType, container: stats.isDirectory(), modified: stats.mtime }
     }
 
     if (stats.isDirectory()) {
@@ -465,7 +465,7 @@ class LDP {
         throw err
       }
       const stream = stringToStream(data)
-      return { stream, contentType, container: true }
+      return { stream, contentType, container: true, modified: stats.mtime }
     } else {
       let chunksize, contentRange, start, end
       if (options.range) {
@@ -487,7 +487,7 @@ class LDP {
           })
           .on('open', function () {
             debug.handlers(`GET -- Reading ${pathLocal}`)
-            return resolve({ stream, contentType, container: false, contentRange, chunksize })
+            return resolve({ stream, contentType, container: false, contentRange, chunksize, modified: stats.mtime })
           })
       }))
     }

lib/webid/lib/get.mjs

@@ -1,19 +1,30 @@
-import { URL } from 'url'
-
-export default function get (webid, callback) {
-  let uri
+import { URL } from 'url'
+import { Agent } from 'undici'
+
+const insecureDispatcher = new Agent({
+  connect: {
+    rejectUnauthorized: false
+  }
+})
+
+export default function get (webid, callback) {
+  let uri
   try {
     uri = new URL(webid)
   } catch (err) {
     return callback(new Error('Invalid WebID URI: ' + webid + ': ' + err.message))
   }
-  const headers = {
-    Accept: 'text/turtle, application/ld+json'
-  }
-  fetch(uri.href, { method: 'GET', headers })
-    .then(async res => {
-      if (!res.ok) {
-        return callback(new Error('Failed to retrieve WebID from ' + uri.href + ': HTTP ' + res.status))
+  const headers = {
+    Accept: 'text/turtle, application/ld+json'
+  }
+  const options = { method: 'GET', headers }
+  if (uri.protocol === 'https:' && process.env.NODE_TLS_REJECT_UNAUTHORIZED === '0') {
+    options.dispatcher = insecureDispatcher
+  }
+  fetch(uri.href, options)
+    .then(async res => {
+      if (!res.ok) {
+        return callback(new Error('Failed to retrieve WebID from ' + uri.href + ': HTTP ' + res.status))
       }
       const contentType = res.headers.get('content-type')
       let body

test/integration/http-test.mjs

@@ -9,9 +9,9 @@ import { assert, expect } from 'chai'
 const __filename = fileURLToPath(import.meta.url)
 const __dirname = path.dirname(__filename)
 
-const suffixAcl = '.acl'
-const suffixMeta = '.meta'
-const server = setupSupertestServer({
+const suffixAcl = '.acl'
+const suffixMeta = '.meta'
+const server = setupSupertestServer({
   live: true,
   dataBrowserPath: 'default',
   root: path.join(__dirname, '../resources'),
@@ -253,13 +253,20 @@ describe('HTTP APIs', function () {
           .expect('content-type', /text\/turtle/)
           .expect('Access-Control-Allow-Origin', 'http://example.com')
           .expect(200, done)
-      })
-    it('should have set Link as resource', function (done) {
-      server.get('/sampleContainer2/example1.ttl')
-        .expect('content-type', /text\/turtle/)
-        .expect('Link', /<http:\/\/www.w3.org\/ns\/ldp#Resource>; rel="type"/)
-        .expect(200, done)
-    })
+      })
+    it('should have set Link as resource', function (done) {
+      server.get('/sampleContainer2/example1.ttl')
+        .expect('content-type', /text\/turtle/)
+        .expect('Link', /<http:\/\/www.w3.org\/ns\/ldp#Resource>; rel="type"/)
+        .expect(200, done)
+    })
+    it('should have set Last-Modified for resource', function (done) {
+      const modified = fs.statSync(path.join(__dirname,
+        '../resources/sampleContainer2/example1.ttl')).mtime.toUTCString()
+      server.get('/sampleContainer2/example1.ttl')
+        .expect('Last-Modified', modified)
+        .expect(200, done)
+    })
     it('should have set Updates-Via to use WebSockets', function (done) {
       server.get('/sampleContainer2/example1.ttl')
         .expect('updates-via', /wss?:\/\//)
@@ -273,13 +280,20 @@ describe('HTTP APIs', function () {
           .expect(hasHeader('describedBy', 'example1.ttl' + suffixMeta))
           .end(done)
       })
-    it('should have set Link as Container/BasicContainer', function (done) {
-      server.get('/sampleContainer2/')
-        .expect('content-type', /text\/turtle/)
-        .expect('Link', /<http:\/\/www.w3.org\/ns\/ldp#BasicContainer>; rel="type"/)
-        .expect('Link', /<http:\/\/www.w3.org\/ns\/ldp#Container>; rel="type"/)
-        .expect(200, done)
-    })
+    it('should have set Link as Container/BasicContainer', function (done) {
+      server.get('/sampleContainer2/')
+        .expect('content-type', /text\/turtle/)
+        .expect('Link', /<http:\/\/www.w3.org\/ns\/ldp#BasicContainer>; rel="type"/)
+        .expect('Link', /<http:\/\/www.w3.org\/ns\/ldp#Container>; rel="type"/)
+        .expect(200, done)
+    })
+    it('should have set Last-Modified for container', function (done) {
+      const modified = fs.statSync(path.join(__dirname,
+        '../resources/sampleContainer2')).mtime.toUTCString()
+      server.get('/sampleContainer2/')
+        .expect('Last-Modified', modified)
+        .expect(200, done)
+    })
     it('should load skin (mashlib) if resource was requested as text/html', function (done) {
       server.get('/sampleContainer2/example1.ttl')
         .set('Accept', 'text/html')
@@ -505,11 +519,18 @@ describe('HTTP APIs', function () {
         .expect('updates-via', /wss?:\/\//)
         .expect(200, done)
     })
-    it('should have set Link as Resource', function (done) {
-      server.head('/sampleContainer2/example1.ttl')
-        .expect('Link', /<http:\/\/www.w3.org\/ns\/ldp#Resource>; rel="type"/)
-        .expect(200, done)
-    })
+    it('should have set Link as Resource', function (done) {
+      server.head('/sampleContainer2/example1.ttl')
+        .expect('Link', /<http:\/\/www.w3.org\/ns\/ldp#Resource>; rel="type"/)
+        .expect(200, done)
+    })
+    it('should have set Last-Modified for resource', function (done) {
+      const modified = fs.statSync(path.join(__dirname,
+        '../resources/sampleContainer2/example1.ttl')).mtime.toUTCString()
+      server.head('/sampleContainer2/example1.ttl')
+        .expect('Last-Modified', modified)
+        .expect(200, done)
+    })
     it('should have set acl and describedBy Links for resource',
       function (done) {
         server.head('/sampleContainer2/example1.ttl')
@@ -523,13 +544,20 @@ describe('HTTP APIs', function () {
           .expect('Content-Type', /text\/turtle/)
           .expect(200, done)
       })
-    it('should have set Link as Container/BasicContainer',
-      function (done) {
-        server.head('/sampleContainer2/')
-          .expect('Link', /<http:\/\/www.w3.org\/ns\/ldp#BasicContainer>; rel="type"/)
-          .expect('Link', /<http:\/\/www.w3.org\/ns\/ldp#Container>; rel="type"/)
-          .expect(200, done)
-      })
+    it('should have set Link as Container/BasicContainer',
+      function (done) {
+        server.head('/sampleContainer2/')
+          .expect('Link', /<http:\/\/www.w3.org\/ns\/ldp#BasicContainer>; rel="type"/)
+          .expect('Link', /<http:\/\/www.w3.org\/ns\/ldp#Container>; rel="type"/)
+          .expect(200, done)
+      })
+    it('should have set Last-Modified for container', function (done) {
+      const modified = fs.statSync(path.join(__dirname,
+        '../resources/sampleContainer2')).mtime.toUTCString()
+      server.head('/sampleContainer2/')
+        .expect('Last-Modified', modified)
+        .expect(200, done)
+    })
     it('should have set acl and describedBy Links for container',
       function (done) {
         server.head('/sampleContainer2/')

test/unit/webid-get-test.mjs

@@ -0,0 +1,70 @@
+import { expect } from 'chai'
+import get from '../../lib/webid/lib/get.mjs'
+
+describe('webid get()', () => {
+  const originalFetch = global.fetch
+  const originalTlsSetting = process.env.NODE_TLS_REJECT_UNAUTHORIZED
+
+  function callGet (webid) {
+    return new Promise((resolve, reject) => {
+      get(webid, (err, body, contentType) => {
+        if (err) {
+          reject(err)
+          return
+        }
+        resolve({ body, contentType })
+      })
+    })
+  }
+
+  afterEach(() => {
+    global.fetch = originalFetch
+    if (originalTlsSetting === undefined) {
+      delete process.env.NODE_TLS_REJECT_UNAUTHORIZED
+    } else {
+      process.env.NODE_TLS_REJECT_UNAUTHORIZED = originalTlsSetting
+    }
+  })
+
+  it('uses an insecure dispatcher for https fetches when TLS verification is disabled', async () => {
+    process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0'
+
+    global.fetch = async (url, options) => {
+      expect(url).to.equal('https://example.com/profile/card#me')
+      expect(options.method).to.equal('GET')
+      expect(options.headers.Accept).to.equal('text/turtle, application/ld+json')
+      expect(options.dispatcher).to.exist
+
+      return {
+        ok: true,
+        headers: {
+          get: () => 'text/turtle'
+        },
+        text: async () => '@prefix ex: <http://example.com/> .'
+      }
+    }
+
+    const { body, contentType } = await callGet('https://example.com/profile/card#me')
+    expect(contentType).to.equal('text/turtle')
+    expect(body).to.include('@prefix ex:')
+  })
+
+  it('does not use an insecure dispatcher when TLS verification is enabled', async () => {
+    delete process.env.NODE_TLS_REJECT_UNAUTHORIZED
+
+    global.fetch = async (url, options) => {
+      expect(options.dispatcher).to.equal(undefined)
+
+      return {
+        ok: true,
+        headers: {
+          get: () => 'text/turtle'
+        },
+        text: async () => 'ok'
+      }
+    }
+
+    const { body } = await callGet('https://example.com/profile/card#me')
+    expect(body).to.equal('ok')
+  })
+})