Sky's Blog
记录 网络 生活
Search
books如何快速有效搜索电子书
status
Published
date
Apr 18, 2021
slug
books
summary
搜集的一些书籍
category
技术分享
tags
书籍
阅读
一些获取电子书的渠道
 
未来世界的幸存者
Article Details
搭建 Proxypool 抓取免费节点网站
status
Published
date
Apr 17, 2021
slug
proxypool
summary
节点网站搭建
category
技术分享
tags
节点

效果演示

演示地址：https://ditan.ml

项目地址

源地址：https://github.com/zu1k/proxypool

部署安装

  • 解压文件并改名proxypool

运行

Article Details
白嫖机场
status
Published
date
Apr 15, 2021
slug
air
summary
提供免费节点的机场整理
category
资源分享
tags
机场
 
 
Article Details
科学上网教程
status
Published
date
Apr 8, 2021
slug
gfw
summary
一些科学上网的方法
category
技术分享
tags
科学上网
简单Socks5科学上网教程
  • 先买个海外服务器
  • 电脑打开Windows Terminal ，输入
$ ssh -NTCD 12345 root@服务器IP
  • 输入密码即可
  • 浏览器或插件设置socks5代理 127.0.0.1:12345
  • 畅通无阻科学上网
 
Article Details
Workers反向代理
status
Published
date
Apr 2, 2021
slug
workers
summary
使用workers进行反向代理的几种方法
category
技术分享
tags
workers
 

反代

  1. 整站
      2. // 替换成你想镜像的站点
const upstream = 'www.google.com'
 
// 如果那个站点有专门的移动适配站点，否则保持和上面一致
const upstream_mobile = 'www.google.com'
 
// 你希望禁止哪些国家访问
const blocked_region = ['RU']
 
// 禁止自访问
const blocked_ip_address = ['0.0.0.0', '127.0.0.1']
 
// 替换成你想镜像的站点
const replace_dict = {
    '$upstream': '$custom_domain',
    '//www.google.com': ''
}
 
//以下内容都不用动
addEventListener('fetch', event => {
    event.respondWith(fetchAndApply(event.request));
})
 
async function fetchAndApply(request) {
 
    const region = request.headers.get('cf-ipcountry').toUpperCase();
    const ip_address = request.headers.get('cf-connecting-ip');
    const user_agent = request.headers.get('user-agent');
 
    let response = null;
    let url = new URL(request.url);
    let url_host = url.host;
 
    if (url.protocol == 'http:') {
        url.protocol = 'https:'
        response = Response.redirect(url.href);
        return response;
    }
 
    if (await device_status(user_agent)) {
        upstream_domain = upstream
    } else {
        upstream_domain = upstream_mobile
    }
 
    url.host = upstream_domain;
 
    if (blocked_region.includes(region)) {
        response = new Response('Access denied: WorkersProxy is not available in your region yet.', {
            status: 403
        });
    } else if(blocked_ip_address.includes(ip_address)){
        response = new Response('Access denied: Your IP address is blocked by WorkersProxy.', {
            status: 403
        });
    } else{
        let method = request.method;
        let request_headers = request.headers;
        let new_request_headers = new Headers(request_headers);
 
        new_request_headers.set('Host', upstream_domain);
        new_request_headers.set('Referer', url.href);
 
        let original_response = await fetch(url.href, {
            method: method,
            headers: new_request_headers
        })
 
        let original_response_clone = original_response.clone();
        let original_text = null;
        let response_headers = original_response.headers;
        let new_response_headers = new Headers(response_headers);
        let status = original_response.status;
 
        new_response_headers.set('access-control-allow-origin', '*');
        new_response_headers.set('access-control-allow-credentials', true);
        new_response_headers.delete('content-security-policy');
        new_response_headers.delete('content-security-policy-report-only');
        new_response_headers.delete('clear-site-data');
 
        const content_type = new_response_headers.get('content-type');
        if (content_type.includes('text/html') && content_type.includes('UTF-8')) {
            original_text = await replace_response_text(original_response_clone, upstream_domain, url_host);
        } else {
            original_text = original_response_clone.body
        }
 
        response = new Response(original_text, {
            status,
            headers: new_response_headers
        })
    }
    return response;
}
 
async function replace_response_text(response, upstream_domain, host_name) {
    let text = await response.text()
 
    var i, j;
    for (i in replace_dict) {
        j = replace_dict[i]
        if (i == '$upstream') {
            i = upstream_domain
        } else if (i == '$custom_domain') {
            i = host_name
        }
 
        if (j == '$upstream') {
            j = upstream_domain
        } else if (j == '$custom_domain') {
            j = host_name
        }
 
        let re = new RegExp(i, 'g')
        text = text.replace(re, j);
    }
    return text;
}
 
async function device_status (user_agent_info) {
    var agents = ["Android", "iPhone", "SymbianOS", "Windows Phone", "iPad", "iPod"];
    var flag = true;
    for (var v = 0; v < agents.length; v++) { if (user_agent_info.indexOf(agents[v]) > 0) {
            flag = false;
            break;
        }
    }
    return flag;
}
  1. 带密码
      2. // 替换成你想镜像的站点
const upstream = 'google.com'
 
// 如果那个站点有专门的移动适配站点，否则保持和上面一致
const upstream_mobile = 'm.google.com'
 
// 密码访问
 
const openAuth = false
const username = 'username'
const password = 'password'
 
// 你希望禁止哪些国家访问
const blocked_region = ['RU']
 
// 禁止自访问
const blocked_ip_address = ['0.0.0.0', '127.0.0.1']
 
// 替换成你想镜像的站点
const replace_dict = {
    '$upstream': '$custom_domain',
    '//google.com': ''
}
 
function unauthorized() {
  return new Response('Unauthorized', {
    headers: {
      'WWW-Authenticate': 'Basic realm="goindex"',
      'Access-Control-Allow-Origin': '*'
    },
    status: 401
  });
}
 
function parseBasicAuth(auth) {
    try {
      return atob(auth.split(' ').pop()).split(':');
    } catch (e) {
      return [];
    }
}
 
function doBasicAuth(request) {
  const auth = request.headers.get('Authorization');
 
  if (!auth || !/^Basic [A-Za-z0-9._~+/-]+=*$/i.test(auth)) {
    return false;
  }
 
  const [user, pass] = parseBasicAuth(auth);
  return user === username && pass === password;
}
 
 
async function fetchAndApply(request) {
  if (request.method === 'OPTIONS') // allow preflight request
    return new Response('', {
      status: 200,
      headers: {
        'Access-Control-Allow-Origin': '*',
        'Access-Control-Allow-Headers': '*',
        'Access-Control-Allow-Methods': 'GET, POST, PUT, HEAD, OPTIONS'
      }
    });
 
  if (openAuth && !doBasicAuth(request)) {
    return unauthorized();
  }
    const region = request.headers.get('cf-ipcountry').toUpperCase();
    const ip_address = request.headers.get('cf-connecting-ip');
    const user_agent = request.headers.get('user-agent');
 
    let response = null;
    let url = new URL(request.url);
    let url_host = url.host;
 
    if (url.protocol == 'http:') {
        url.protocol = 'https:'
        response = Response.redirect(url.href);
        return response;
    }
 
    if (await device_status(user_agent)) {
        upstream_domain = upstream
    } else {
        upstream_domain = upstream_mobile
    }
 
    url.host = upstream_domain;
 
    if (blocked_region.includes(region)) {
        response = new Response('Access denied: WorkersProxy is not available in your region yet.', {
            status: 403
        });
    } else if(blocked_ip_address.includes(ip_address)){
        response = new Response('Access denied: Your IP address is blocked by WorkersProxy.', {
            status: 403
        });
    } else{
        let method = request.method;
        let request_headers = request.headers;
        let new_request_headers = new Headers(request_headers);
 
        new_request_headers.set('Host', upstream_domain);
        new_request_headers.set('Referer', url.href);
 
        let original_response = await fetch(url.href, {
            method: method,
            headers: new_request_headers
        })
 
        let original_response_clone = original_response.clone();
        let original_text = null;
        let response_headers = original_response.headers;
        let new_response_headers = new Headers(response_headers);
        let status = original_response.status;
 
        new_response_headers.set('access-control-allow-origin', '*');
        new_response_headers.set('access-control-allow-credentials', true);
        new_response_headers.delete('content-security-policy');
        new_response_headers.delete('content-security-policy-report-only');
        new_response_headers.delete('clear-site-data');
 
        const content_type = new_response_headers.get('content-type');
        if (content_type.includes('text/html') && content_type.includes('UTF-8')) {
            original_text = await replace_response_text(original_response_clone, upstream_domain, url_host);
        } else {
            original_text = original_response_clone.body
        }
 
        response = new Response(original_text, {
            status,
            headers: new_response_headers
        })
    }
    return response;
}
 
addEventListener('fetch', event => {
    event.respondWith(fetchAndApply(event.request).catch(err => {
      console.error(err);
      new Response(JSON.stringify(err.stack), {
        status: 500,
        headers: {
          'Content-Type': 'application/json'
        }
      });
    }));
})
 
 
async function replace_response_text(response, upstream_domain, host_name) {
    let text = await response.text()
 
    var i, j;
    for (i in replace_dict) {
        j = replace_dict[i]
        if (i == '$upstream') {
            i = upstream_domain
        } else if (i == '$custom_domain') {
            i = host_name
        }
 
        if (j == '$upstream') {
            j = upstream_domain
        } else if (j == '$custom_domain') {
            j = host_name
        }
 
        let re = new RegExp(i, 'g')
        text = text.replace(re, j);
    }
    return text;
}
 
async function device_status (user_agent_info) {
    var agents = ["Android", "iPhone", "SymbianOS", "Windows Phone", "iPad", "iPod"];
    var flag = true;
    for (var v = 0; v < agents.length; v++) { if (user_agent_info.indexOf(agents[v]) > 0) {
            flag = false;
            break;
        }
    }
    return flag;
}
  1. 部分代理
      2. // 你要镜像的网站.
const upstream = 'objectstorage.ap-seoul-1.oraclecloud.com'
 
// 镜像网站的目录，比如你想镜像某个网站的二级目录则填写二级目录的目录名，镜像 google 用不到，默认即可.
const upstream_path = '/test/'
 
// 镜像站是否有手机访问专用网址，没有则填一样的.
const upstream_mobile = 'objectstorage.ap-seoul-1.oraclecloud.com'
 
// 屏蔽国家和地区.
const blocked_region = ['']
 
// 屏蔽 IP 地址.
const blocked_ip_address = ['0.0.0.0', '127.0.0.1']
 
// 镜像站是否开启 HTTPS.
const https = true
 
// 文本替换.填你要镜像的网站
const replace_dict = {
    '$upstream': '$custom_domain',
    '//objectstorage.ap-seoul-1.oraclecloud.com': ''
}
 
// 以下保持默认，不要动
addEventListener('fetch', event => {
    event.respondWith(fetchAndApply(event.request));
})
 
async function fetchAndApply(request) {
 
    const region = request.headers.get('cf-ipcountry').toUpperCase();
    const ip_address = request.headers.get('cf-connecting-ip');
    const user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36"
 
    let response = null;
    let url = new URL(request.url);
    let url_hostname = url.hostname;
 
    if (https == true) {
        url.protocol = 'https:';
    } else {
        url.protocol = 'http:';
    }
 
    if (await device_status(user_agent)) {
        var upstream_domain = upstream;
    } else {
        var upstream_domain = upstream_mobile;
    }
 
    url.host = upstream_domain;
    if (url.pathname == '/') {
        url.pathname = upstream_path;
    } else {
        url.pathname = upstream_path + url.pathname;
    }
 
    if (blocked_region.includes(region)) {
        response = new Response('Access denied: WorkersProxy is not available in your region yet.', {
            status: 403
        });
    } else if (blocked_ip_address.includes(ip_address)) {
        response = new Response('Access denied: Your IP address is blocked by WorkersProxy.', {
            status: 403
        });
    } else {
        let method = request.method;
        let request_headers = request.headers;
        let new_request_headers = new Headers(request_headers);
 
        new_request_headers.set('Host', url.hostname);
        new_request_headers.set('Referer', url.hostname);
 
        let original_response = await fetch(url.href, {
            method: method,
            headers: new_request_headers
        })
 
        let original_response_clone = original_response.clone();
        let original_text = null;
        let response_headers = original_response.headers;
        let new_response_headers = new Headers(response_headers);
        let status = original_response.status;
 
        new_response_headers.set('access-control-allow-origin', '*');
        new_response_headers.set('access-control-allow-credentials', true);
        new_response_headers.delete('content-security-policy');
        new_response_headers.delete('content-security-policy-report-only');
        new_response_headers.delete('clear-site-data');
 
        const content_type = new_response_headers.get('content-type');
        if (content_type.includes('text/html') && content_type.includes('UTF-8')) {
            original_text = await replace_response_text(original_response_clone, upstream_domain, url_hostname);
        } else {
            original_text = original_response_clone.body
        }
 
        response = new Response(original_text, {
            status,
            headers: new_response_headers
        })
    }
    return response;
}
 
async function replace_response_text(response, upstream_domain, host_name) {
    let text = await response.text()
 
    var i, j;
    for (i in replace_dict) {
        j = replace_dict[i]
        if (i == '$upstream') {
            i = upstream_domain
        } else if (i == '$custom_domain') {
            i = host_name
        }
 
        if (j == '$upstream') {
            j = upstream_domain
        } else if (j == '$custom_domain') {
            j = host_name
        }
 
        let re = new RegExp(i, 'g')
        text = text.replace(re, j);
    }
    return text;
}
 
 
async function device_status(user_agent_info) {
    var agents = ["Android", "iPhone", "SymbianOS", "Windows Phone", "iPad", "iPod"];
    var flag = true;
    for (var v = 0; v < agents.length; v++) {
        if (user_agent_info.indexOf(agents[v]) > 0) {
            flag = false;
            break;
        }
    }
    return flag;
}
       
带密码验证
// 替换成你想镜像的站点
const upstream = 'www.google.com'

// 如果那个站点有专门的移动适配站点，否则保持和上面一致
const upstream_mobile = 'm.google.com'

// 密码访问

const openAuth = false
const username = 'zhaoip'
const password = 'xyz'

// 你希望禁止哪些国家访问
const blocked_region = ['KP']

// 禁止自访问
const blocked_ip_address = ['0.0.0.0', '127.0.0.1']

// 替换成你想镜像的站点
const replace_dict = {
    '$upstream': '$custom_domain',
    '//google.com': ''
}

function unauthorized() {
  return new Response('Unauthorized', {
    headers: {
      'WWW-Authenticate': 'Basic realm="goindex"',
      'Access-Control-Allow-Origin': '*'
    },
    status: 401
  });
}

function parseBasicAuth(auth) {
    try {
      return atob(auth.split(' ').pop()).split(':');
    } catch (e) {
      return [];
    }
}

function doBasicAuth(request) {
  const auth = request.headers.get('Authorization');

  if (!auth || !/^Basic [A-Za-z0-9._~+/-]+=*$/i.test(auth)) {
    return false;
  }

  const [user, pass] = parseBasicAuth(auth);
  return user === username && pass === password;
}


async function fetchAndApply(request) {
  if (request.method === 'OPTIONS') // allow preflight request
    return new Response('', {
      status: 200,
      headers: {
        'Access-Control-Allow-Origin': '*',
        'Access-Control-Allow-Headers': '*',
        'Access-Control-Allow-Methods': 'GET, POST, PUT, HEAD, OPTIONS'
      }
    });

  if (openAuth && !doBasicAuth(request)) {
    return unauthorized();
  }
    const region = request.headers.get('cf-ipcountry').toUpperCase();
    const ip_address = request.headers.get('cf-connecting-ip');
    const user_agent = request.headers.get('user-agent');

    let response = null;
    let url = new URL(request.url);
    let url_host = url.host;

    if (url.protocol == 'http:') {
        url.protocol = 'https:'
        response = Response.redirect(url.href);
        return response;
    }

    if (await device_status(user_agent)) {
        upstream_domain = upstream
    } else {
        upstream_domain = upstream_mobile
    }

    url.host = upstream_domain;

    if (blocked_region.includes(region)) {
        response = new Response('Access denied: WorkersProxy is not available in your region yet.', {
            status: 403
        });
    } else if(blocked_ip_address.includes(ip_address)){
        response = new Response('Access denied: Your IP address is blocked by WorkersProxy.', {
            status: 403
        });
    } else{
        let method = request.method;
        let request_headers = request.headers;
        let new_request_headers = new Headers(request_headers);

        new_request_headers.set('Host', upstream_domain);
        new_request_headers.set('Referer', url.href);

        let original_response = await fetch(url.href, {
            method: method,
            headers: new_request_headers
        })

        let original_response_clone = original_response.clone();
        let original_text = null;
        let response_headers = original_response.headers;
        let new_response_headers = new Headers(response_headers);
        let status = original_response.status;

        new_response_headers.set('access-control-allow-origin', '*');
        new_response_headers.set('access-control-allow-credentials', true);
        new_response_headers.delete('content-security-policy');
        new_response_headers.delete('content-security-policy-report-only');
        new_response_headers.delete('clear-site-data');

        const content_type = new_response_headers.get('content-type');
        if (content_type.includes('text/html') && content_type.includes('UTF-8')) {
            original_text = await replace_response_text(original_response_clone, upstream_domain, url_host);
        } else {
            original_text = original_response_clone.body
        }

        response = new Response(original_text, {
            status,
            headers: new_response_headers
        })
    }
    return response;
}

addEventListener('fetch', event => {
    event.respondWith(fetchAndApply(event.request).catch(err => {
      console.error(err);
      new Response(JSON.stringify(err.stack), {
        status: 500,
        headers: {
          'Content-Type': 'application/json'
        }
      });
    }));
})


async function replace_response_text(response, upstream_domain, host_name) {
    let text = await response.text()

    var i, j;
    for (i in replace_dict) {
        j = replace_dict[i]
        if (i == '$upstream') {
            i = upstream_domain
        } else if (i == '$custom_domain') {
            i = host_name
        }

        if (j == '$upstream') {
            j = upstream_domain
        } else if (j == '$custom_domain') {
            j = host_name
        }

        let re = new RegExp(i, 'g')
        text = text.replace(re, j);
    }
    return text;
}

async function device_status (user_agent_info) {
    var agents = ["Android", "iPhone", "SymbianOS", "Windows Phone", "iPad", "iPod"];
    var flag = true;
    for (var v = 0; v < agents.length; v++) {
        if (user_agent_info.indexOf(agents[v]) > 0) {
            flag = false;
            break;
        }
    }
    return flag;
}
Article Details
fastapi学习笔记
status
Published
date
Mar 16, 2021
slug
fastapi
summary
fastapi学习笔记
category
学习思考
tags
Fast API
简单运行
from fastapi import FastAPI
app = FastAPI()
@app.get('/')
def index():
		return {'hello':'world'}
一个简单的项目

main.py

from fastapi import FastAPI  
import uvicorn
import msg,ip

app = FastAPI()
app.include_router(msg.router, prefix="/msg")
app.include_router(ip.router, prefix="/ip")

@app.get('/')
async def index():
    return '首页'


if __name__ == '__main__':
    uvicorn.run(app='main:app',host='0.0.0.0',port=12345,reload=True,debug=True)

ip.py

from fastapi import APIRouter
import requests
from bs4 import BeautifulSoup
import json
from random import choice
router = APIRouter()
def api():
    url = 'http://proxylist.fatezero.org/proxy.list'
    headers = {"User-Agent":"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36"}
    res = requests.get(url,headers=headers)
    res.encoding = "utf-8"
    s = res.text
    data = []
    for i in s.split('\n'):
        try:
            ip = str(json.loads(i)['host'])+':'+str(json.loads(i)['port'])
            msg = {"ip":ip,"country":str(json.loads(i)['country']),"type":str(json.loads(i)['type']),"response_time":str(json.loads(i)['response_time']),'weixin':'zhaoipxyz'}
            data.append(msg)
        except:
            pass
    return data

def choiceone():
    ip = choice(api())
    return ip

@router.get('/')
async def msg():
    return choiceone()

msg.py

from peewee import *
import requests,time
from bs4 import BeautifulSoup

db = PostgresqlDatabase('kfnmhpuuszajss:4bb62ccc1b91797c653df18b169ad38f541489c9ed56d0770ae67823b5663227@ec2-54-156-149-189.compute-1.amazonaws.com:5432/d14bupst16squ7')

class BaseModel(Model):
    class Meta:
        database = db

class Share(BaseModel):
    title = CharField(verbose_name='标题', null=False, index=True)
    link = CharField(verbose_name='链接', null=False)
    site = CharField(verbose_name='来源站点', null=False)
    pubtime = DateTimeField(verbose_name='发布时间', null=False, default=time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()))
    class Meta:
        table_name = 'share'
try:
    Share.create_table()
except:
    print('表已存在')

class Xianbao:
    def __init__(self):
        self.headers = {
            'User-Agent': 'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.167 Safari/537.36'
            }

    def wgzj(self):
        site = '网购之家'
        r = requests.get('http://www.wgzj.cn/bbs/forum-2-1.html',headers=self.headers)
        html = r.text
        bs = BeautifulSoup(html,'html.parser')
        links= bs.findAll('a',class_='s xst')
        for link in links:
            title= link.text
            url = 'http://www.wgzj.cn/bbs/' + link['href']
            if Share.get(link==url).frist():
                print(title,'数据已存在！')
            else:
                Share.create(title=title,site=site,link=link)
                print(title,'采集成功！')
            

    def kxd(self):
        site ='科学刀'
        r =requests.get('https://kxdao.net/forum-42-1.html',headers=self.headers)
        html = r.text
        bs = BeautifulSoup(html,'html.parser')
        links = bs.findAll('a',class_='s xst')
        for link in links:
            title = link.text
            url = link['href']
            if Share.get(link==url).frist():
                print(title,'数据已存在！')
            else:
                Share.create(title=title,site=site,link=link)
                print(title,'采集成功！')

    def leyu(self):
        site = '乐愚社区'
        r = requests.get('https://bbs.leyuz.net/f/xianbao', headers=self.headers)
        html = r.text
        bs = BeautifulSoup(html,'html.parser')
        links = bs.findAll('a',class_="tag-title")
        for link in links:
            title = link.text
            url = 'https://bbs.leyuz.net' + link['href']
            if Share.get(link==url).frist():
                print(title,'数据已存在！')
            else:
                Share.create(title=title,site=site,link=link)
                print(title,'采集成功！')

    def xb0818(self):
        site = '线报0818'
        r = requests.get('http://www.0818tuan.com/list-1-0.html', headers=self.headers)
        html = r.text
        bs = BeautifulSoup(html, 'html.parser')
        links = bs.findAll('a', href=re.compile(r"/xbhd/\d+\.html"))
        for link in links:
            title = link.text
            url = "http://www.0818tuan.com" + link['href']
            if Share.get(link==url).frist():
                print(title,'数据已存在！')
            else:
                Share.create(title=title,site=site,link=link)
                print(title,'采集成功！')
IP检测
 
Article Details
12345