Skip to content

Commit

Permalink
added cloudflare support.
Browse files Browse the repository at this point in the history
  • Loading branch information
@netptop
netptop committed Sep 17, 2020
1 parent dcde239 commit e0ed919
Show file tree
Hide file tree
Showing 14 changed files with 544 additions and 43 deletions.
2 changes: 2 additions & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,3 +8,5 @@ yarn.lock
.history
.umirc.local.js
.now
bundle.js
worker/
144 changes: 109 additions & 35 deletions Proxy.js
View file
Original file line number Diff line number Diff line change
@@ -1,19 +1,34 @@
var express = require('express');
var proxy = require('http-proxy-middleware');
const zlib = require("zlib")
const fs = require("fs")
const zlib = require('zlib')
const parse = require('url-parse')
const cookiejar = require('cookiejar')
const iconv = require('iconv-lite')
const pako = require('pako')
const {logSave, logGet, logClear} = require('./logger')
const {CookieAccessInfo, CookieJar, Cookie} = cookiejar
const ProxyMiddleware = require('http-proxy-middleware')

function countUtf8Bytes(s) {
var b = 0, i = 0, c
for(;c=s.charCodeAt(i++);b+=c>>11?3:c>>7?2:1);
return b
}

function uint8ArrayConcat(arrays) { // for cloudflare
// sum of individual array lengths
let totalLength = arrays.reduce((acc, value) => acc + value.length, 0);
if (!arrays.length) return null;

let result = new Uint8Array(totalLength);
// for each array - copy it over result
// next array is copied right after the previous one
let length = 0;
for(let array of arrays) {
result.set(array, length);
length += array.length;
}

return result;
}

var contentTypeIsText = (headers) => {
if (!headers["content-type"] ||
headers["content-type"].indexOf('text/') !== -1 ||
Expand Down Expand Up @@ -62,7 +77,7 @@ var redirect2HomePage = function({res, httpprefix, serverName,} ) {
res.status(302).send(``)
}

let getHostFromReq = ({req, serverName}) => { //return target
let getHostFromReq = ({req, serverName}) => { // return target
// url: http://127.0.0.1:8011/https/www.youtube.com/xxx/xxx/...
let https_prefix = '/https/'
let http_prefix = '/http/'
Expand Down Expand Up @@ -116,7 +131,9 @@ let getHostFromReq = ({req, serverName}) => { //return target
}
})
}
if (host === '') {
let localServeList = ['/index.html', '/', '/favicon.png']
if (host === '' ||
(host === serverName && !localServeList.includes(req.url))) {
if (originalHost !== '') {
httpType = originalHost.split('/')[0]
host = originalHost.split('/')[1]
Expand All @@ -127,12 +144,9 @@ let getHostFromReq = ({req, serverName}) => { //return target
return {host, httpType}
}


let Proxy = ({blockedSites, urlModify, httpprefix, serverName, port, cookieDomainRewrite, locationReplaceMap302, regReplaceMap, siteSpecificReplace, pathReplace}) => {
let stream = fs.createWriteStream("web-records.csv", {flags:'a'})
let handleRespond = ({req, res, body, gbFlag}) => { // text file
let myRe
let {host, httpType} = getHostFromReq({req, serverName})
let Proxy = ({ProxyMiddleware, blockedSites, urlModify, httpprefix, serverName, port, cookieDomainRewrite, locationReplaceMap302, regReplaceMap, siteSpecificReplace, pathReplace}) => {
// let stream = fs.createWriteStream("web-records.csv", {flags:'a'})
var locationMod302 = ({res, serverName, httpprefix, host, httpType}) => {
let location = res.getHeaders()['location']
if (res.statusCode == '301' || res.statusCode == '302' || res.statusCode == '303' ||res.statusCode == '307' || res.statusCode == '308') {
location = locationReplaceMap302({location, serverName, httpprefix, host, httpType})
Expand All @@ -141,9 +155,16 @@ let Proxy = ({blockedSites, urlModify, httpprefix, serverName, port, cookieDomai
res.setHeader('location', location)
} catch(e) {
logSave(`error: ${e}`)
return
return false
}
// return
}
return true
}
let handleRespond = ({req, res, body, gbFlag}) => { // text file
let myRe
let {host, httpType} = getHostFromReq({req, serverName})
if (locationMod302({res, serverName, httpprefix, host, httpType}) === false) {
return
}
// logSave(`HandleRespond(), req.url:${req.url}, req.headers:${JSON.stringify(req.headers)}`)
for(let key in regReplaceMap) {
Expand Down Expand Up @@ -190,15 +211,23 @@ let Proxy = ({blockedSites, urlModify, httpprefix, serverName, port, cookieDomai
}
res.statusCode = '302'
}
logSave(`6`)
body = zlib.gzipSync(body) //19ms
logSave(`5 after replacment,displayed string, body.length:${body.length}`)
// logSave(`5 after replacment,displayed string: ${body}`)
if (process.env.cloudflare === 'true') { // in cloudflare environment
let enc = new TextEncoder()
body = enc.encode(body)
logSave(`6 cloudflare, after encoding, uint8array, body.length:${body.length}`) // body is utf-8 uint8Array now
} else { //node environment
body = zlib.gzipSync(body) //body is Buffer
}
logSave(`7`)
try {
res.setHeader('content-encoding', 'gzip');
logSave(`handleRespond: res.statusCode:${res.statusCode}, res.headers:${JSON.stringify(res.getHeaders())}`)
if (req.headers['debugflag']==='true') {
res.removeHeader('content-encoding')
res.setHeader('content-type','text/plain')
body=logGet()
body = logGet()
}
res.end(body);
} catch(e) {
Expand All @@ -216,7 +245,7 @@ let Proxy = ({blockedSites, urlModify, httpprefix, serverName, port, cookieDomai
return target
}

let p = proxy({
let p = ProxyMiddleware({
target: `https://www.google.com`,
router,
/*
Expand Down Expand Up @@ -256,14 +285,18 @@ let Proxy = ({blockedSites, urlModify, httpprefix, serverName, port, cookieDomai
let bodyLength = 0
let endFlag = false
proxyRes.on('data', function(data) {
// body = Buffer.concat([body, data]);
if (endFlag === true) {
return // don't have to push it to bodyList
}
bodyLength += data.length
bodyLength += data.length // data is Uint8Array for cloueflare, and Buffer for node environment
bodyList.push(data)
if (bodyLength >= 2500000 && contentTypeIsText(proxyRes.headers) !== true) {
let body = Buffer.concat(bodyList)
let body
if (process.env.cloudflare === 'true') { // in node environment
body = uint8ArrayConcat(bodyList) // body is Uint8Array for cloueflare
} else {
body = Buffer.concat(bodyList) // body is Buffer for node environment
}
let fwdStr = req.headers['X-Forwarded-For'] || req.headers['x-forwarded-for']
let contentType = proxyRes.headers['content-type']
let contentLen = proxyRes.headers['content-length']
Expand All @@ -280,32 +313,58 @@ let Proxy = ({blockedSites, urlModify, httpprefix, serverName, port, cookieDomai
if (endFlag === true) {
return
}
let body = Buffer.concat(bodyList)
console.log(`on end 1, bodyList.length:${bodyList.length}`)
let body
if (process.env.cloudflare === 'true') {
body = uint8ArrayConcat(bodyList) // body is Uint8Array
if (!body) { // if body is null, end the stream.
locationMod302({res, serverName, httpprefix, host, httpType})
res.end(body);
return
}
} else { // in node environment
body = Buffer.concat(bodyList) // body is Buffer
}
console.log(`on end 2, body.length:${body.length}`)
let gbFlag = false
if (proxyRes.headers["content-encoding"] === 'gzip' ||
proxyRes.headers["content-encoding"] === 'br') { // gzip/br encoding
let gunzipped
try {
if (proxyRes.headers["content-encoding"] === 'br') {
gunzipped = zlib.brotliDecompressSync(body)
logSave(`zlib.brotli...`)
} else { //gzip
gunzipped = zlib.gunzipSync(body)
if (process.env.cloudflare === 'true') { // in cloudflare environment
console.log('cloudflare environment')
gunzipped = body // in cloudflare, we have gzip
} else {
console.log('node environment')
if (proxyRes.headers["content-encoding"] === 'br') {
gunzipped = zlib.brotliDecompressSync(body)
logSave(`zlib.brotli...`)
} else { //gzip
gunzipped = zlib.gunzipSync(body)
logSave(`zlib.gunzip...`)
}
}
} catch(e) {
// res.status(404).send(`{"error": "${e}"}`)
console.log(`error2:${e}`)
return
}
console.log(`on end 3`)
if (contentTypeIsText(proxyRes.headers) === true) { //gzip and text
console.log(`on end 4`)
if (!gunzipped) {
// res.status(404).send(`{"error":"failed unzip"}`)
redirect2HomePage({res, httpprefix, serverName,})
return
}
logSave(`utf-8 text...`)
logSave(`utf-8 text, gunzipped.length:${gunzipped.length}`)
let originBody = gunzipped
body = gunzipped.toString('utf-8');
if (process.env.cloudflare === 'true') { // in cloudflare environment
body = new TextDecoder().decode(gunzipped) // gunzipped.toString('utf-8');
logSave(`after decode, displayed string, body.length:${body.length}`)
} else {
body = gunzipped.toString('utf-8')
}
let searchBody = body.slice(0, 1000)
if (searchBody.indexOf('="text/html; charset=gb') !== -1 ||
searchBody.indexOf(' charset="gb') !== -1 ||
Expand All @@ -317,15 +376,16 @@ let Proxy = ({blockedSites, urlModify, httpprefix, serverName, port, cookieDomai
let fwdStr = req.headers['X-Forwarded-For'] || req.headers['x-forwarded-for'] || ''
if (proxyRes.statusCode === 200 && proxyRes.headers["content-type"] &&
proxyRes.headers["content-type"].indexOf('text/html') !== -1) {
saveRecord({stream, fwdStr, req, host, pktLen:body.length})
// saveRecord({stream, fwdStr, req, host, pktLen:body.length})
}
if (proxyRes.statusCode === 200 && req.url.indexOf('/sw.js') !== -1) {
// fetching sw.js
res.setHeader('service-worker-allowed','/')
}
handleRespond({req, res, body, gbFlag})
handleRespond({req, res, body, gbFlag}) // body is a displayed string
} else { // gzip and non-text
// console.log(`2========>${logGet()}`)
console.log(`on end 5`)
let fwdStr = req.headers['X-Forwarded-For'] || req.headers['x-forwarded-for']
let contentType = proxyRes.headers['content-type']
let contentLen = proxyRes.headers['content-length']
Expand All @@ -340,7 +400,11 @@ let Proxy = ({blockedSites, urlModify, httpprefix, serverName, port, cookieDomai
contentTypeIsText(proxyRes.headers) === true) { // text with non gzip encoding
logSave(`utf-8 text...`)
let originBody = body
body = body.toString('utf-8');
if (process.env.cloudflare === 'true') { // in cloudflare environment
body = new TextDecoder().decode(body) // Uint8Array(utf-8 arrayBuffer) toString('utf-8')
} else { // node environment
body = body.toString('utf-8');
}
if (body.indexOf('="text/html; charset=gb') !== -1 ||
body.indexOf(' charset="gb') !== -1 ||
body.indexOf('=\'text/html; charset=gb') !== -1) {
Expand All @@ -355,16 +419,22 @@ let Proxy = ({blockedSites, urlModify, httpprefix, serverName, port, cookieDomai
let contentType = proxyRes.headers['content-type']
let contentLen = proxyRes.headers['content-length']
console.log(`end,route:${fwdStr}, content-type:${contentType},length:${bodyLength}, content-length:${contentLen}, ${host}`)
res.end(body)
if (process.env.cloudflare === 'true') { // in cloudflare environment
res.end(Uint8Array.from(body))
} else { // node environment
res.end(body)
}
}
})
const setCookieHeaders = proxyRes.headers['set-cookie'] || []
console.log(`1`)
let datestr = ''
if (setCookieHeaders.length > 0) {
let date = new Date
date.setDate(date.getDate() + 1) // 一天之后过期
datestr = date.toUTCString()
}
console.log(`2, setCookieHeaders:${JSON.stringify(setCookieHeaders)}`)
const modifiedSetCookieHeaders = setCookieHeaders
.map(str => new cookiejar.Cookie(str))
.map(cookie => {
Expand All @@ -378,6 +448,7 @@ let Proxy = ({blockedSites, urlModify, httpprefix, serverName, port, cookieDomai
return cookie
})
.map(cookie => cookie.toString())
console.log(`3`)
let cookie_originalHost= new cookiejar.Cookie()
cookie_originalHost.name = 'ORIGINALHOST'
cookie_originalHost.value = `${httpType}/${host}`
Expand All @@ -386,6 +457,7 @@ let Proxy = ({blockedSites, urlModify, httpprefix, serverName, port, cookieDomai
cookie_originalHost.path = `/`
cookie_originalHost.secure = false
modifiedSetCookieHeaders.push(cookie_originalHost.toString())
console.log(`4`)
proxyRes.headers['set-cookie'] = modifiedSetCookieHeaders
Object.keys(proxyRes.headers).forEach(function (key) {
if (key === 'content-security-policy' ||
Expand Down Expand Up @@ -469,9 +541,11 @@ let Proxy = ({blockedSites, urlModify, httpprefix, serverName, port, cookieDomai
})
proxyReq.setHeader('Accept-Encoding', 'gzip')
proxyReq.setHeader('referer', host)
console.log(`host=${host}`)
if (host.indexOf('youtube.com') !== -1) {
proxyReq.setHeader('User-Agent', `Opera/7.50 (Windows XP; U)`)
// proxyReq.setHeader('User-Agent', `Opera/7.50 (Windows XP; U)`)
// proxyReq.setHeader('User-Agent', `Opera/9.80 (Android 4.1.2; Linux; Opera Mobi/ADR-1305251841) Presto/2.11.355 Version/12.10`)
proxyReq.setHeader('User-Agent', `Mozilla/5.0 (Linux; Android 6.0.1; Lenovo-A6020l36 Build/MMB29M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.93 Mobile Safari/537.36`)
}
logSave(`req host:${host}, req.url:${req.url}, proxyReq.query:${proxyReq.query} proxyReq.path:${proxyReq.path}, proxyReq.url:${proxyReq.url} proxyReq headers:${JSON.stringify(proxyReq.getHeaders())}`)
if(host === '' || !host) {
Expand Down
3 changes: 2 additions & 1 deletion api/main.js
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,13 @@
const fs = require('fs')
const path = require('path')
const ProxyMiddleware = require('http-proxy-middleware')
var Proxy = require('../Proxy')

let { blockedSites, urlModify, httpprefix, serverName, port, locationReplaceMap302, regReplaceMap, siteSpecificReplace, pathReplace } = require('../config')

let cookieDomainRewrite = serverName

let proxy = Proxy({ blockedSites, urlModify, httpprefix, serverName, port, cookieDomainRewrite, locationReplaceMap302, regReplaceMap, siteSpecificReplace, pathReplace })
let proxy = Proxy({ ProxyMiddleware, blockedSites, urlModify, httpprefix, serverName, port, cookieDomainRewrite, locationReplaceMap302, regReplaceMap, siteSpecificReplace, pathReplace })
export default (req, res) => {
const dirPath = path.join(__dirname + '/..', req.url)
console.log(`x-forward-for:${req.headers['x-forwarded-for']}, req.url:${req.url}`)
Expand Down
93 changes: 93 additions & 0 deletions build/worker.js
View file

Large diffs are not rendered by default.

Loading

0 comments on commit e0ed919

Please sign in to comment.