子域名搜集脚本

目录结构：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
dict:
	dict.txt 字典
result：数据存储
	test.com:
		brute.json
		crt.json
		ilink.json
utils:
	brutedns.py：字典爆破
	Crt.py：crt页面爬取
	ilink.py:ilink页面爬取
	__init__.py
config.py:配置文件
common.py:方法
domains.py：执行文件
test.com.log:最终结果
运行方法：
python domains.py -d xxx.com
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
# brutedns.py
# coding=utf-8

import dns.resolver
import threading
from Queue import Queue
from config import *
import sys


class BruteDns(object):
    def __init__(self, domain):
        self.domain = domain
        self.thread_count = thread_count
        self.queue = Queue()
        self.result = []

    def run(self):
        with open('dict/dict.txt') as f:
            for i in f:
                self.queue.put(i.rstrip() + '.' + self.domain)
        total = self.queue.qsize()
        threads = []

        for i in xrange(self.thread_count):
            threads.append(self.BruteRun(self.queue, self.result, total))
        for i in threads:
            i.start()
        for i in threads:
            i.join()

        return list(set(self.result))

    class BruteRun(threading.Thread):
        def __init__(self, queue, result, total):
            threading.Thread.__init__(self)
            self._queue = queue
            self.result = result
            self.total = total

        def run(self):
            while not self._queue.empty():
                sub = self._queue.get_nowait()
                try:
                    self.msg()
                    results = dns.resolver.query(sub, 'A')
                    if results.response.answer:
                        self.result.append(sub)
                except Exception, e:
                    pass

        def msg(self):
            done_count = float(self.total - self._queue.qsize())
            all_count = float(self.total)
            found_count = len(self.result)
            msg = '\t[-]Last {} | Complete {:.2f}% | Found {}\t'.format(self._queue.qsize(),
                                                                        (done_count / all_count) * 100, found_count)
            sys.stdout.write('\r' + msg)
            sys.stdout.flush()
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
# Crt.py
# coding=utf-8

from common import http_requests_get, is_domain
import re


class Crt(object):
    def __init__(self, domain):
        self.domain = domain
        self.site = 'https://crt.sh/?q=%25.'
        self.result = []

    def run(self):
        url = self.site + self.domain
        try:
            r = http_requests_get(url=url)
            results = re.findall('</TD>\n    <TD>(.*?)</TD>\n    <TD><A', r.text, re.S)
            for result in results:
                if is_domain(result):
                    self.result.append(result)
            return list(set(self.result))
        except:
            return self.result
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
#ilink.py
# coding=utf-8

from common import http_requests_post, is_domain
import re


class ILink(object):
    def __init__(self, domain):
        self.domain = domain
        self.site = 'http://i.links.cn/subdomain/'
        self.result = []

    def run(self):
        payload = {
            'domain': self.domain,
            'b2': 1,
            'b3': 1,
            'b4': 1,
        }
        try:
            r = http_requests_post(self.site, payload)
            results = re.findall('value="http://(.*?)"><input', r.text)
            for result in results:
                if is_domain(result):
                    self.result.append(result)
            return list(set(self.result))
        except:
            return self.result
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
# common.py
# coding=utf-8
from config import *
import re

import requests.packages.urllib3

requests.packages.urllib3.disable_warnings()

import requests
import urllib3
import json


def http_requests_get(url, allow_redirects=allow_redirects):
    try:
        result = requests.get(
            url=url,
            headers=headers,
            timeout=timeout,
            allow_redirects=allow_redirects,
            verify=allow_ssl_verify)
        return result
    except Exception, e:
        return requests.models.Response()
        # return e


def http_requests_post(url, payload, allow_redirects=allow_redirects):
    try:
        result = requests.get(
            url=url,
            data=payload,
            headers=headers,
            timeout=timeout,
            allow_redirects=allow_redirects,
            verify=allow_ssl_verify)
        return result
    except Exception, e:
        return requests.models.Response()
        # return e


def is_domain(domain):
    domain_regex = re.compile(
        r'(?:[A-Z0-9_](?:[A-Z0-9-_]{0,247}[A-Z0-9])?\.)+(?:[A-Z]{2,6}|[A-Z0-9-]{2,}(?<!-))\Z', re.IGNORECASE)
    return True if domain_regex.match(domain) else False


def save_result(filename, result):
    try:
        f = open(filename, 'w')
        json.dump(result, f, indent=4)
    except Exception, e:
        print e


def read_json(filename):
    try:
        f = open(filename, 'r')
        datas = json.load(f)
        return datas
    except Exception, e:
        print e
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
# config.py
# coding=utf-8
import sys
import random

# 是否开启https服务的证书校验
allow_ssl_verify = False
# 线程数
thread_count = 50
# timeout
timeout = 10
# 是否允许重定向
allow_redirects = True
# 是否允许继承http Request类的session支持，在发出的所有请求之间保存cookies
allow_http_session = True
# 是否允许随机useragent
allow_random_useragent = False
# 是否允许随机x_forward_for头
allow_random_x_forward = False

USER_AGENTS = [
    'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36',
    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/535.20 (KHTML, like Gecko) Chrome/19.0.1036.7 Safari/535.20',
]


def random_useragent(condition=False):
    if condition:
        return random.choice(USER_AGENTS)
    else:
        return USER_AGENTS[0]


def random_x_forward_for(condition=False):
    if condition:
        return '%d.%d.%d.%d' % (
            random.randint(1, 254), random.randint(1, 254), random.randint(1, 254), random.randint(1, 254))
    else:
        return '8.8.8.8'


headers = {
    'User-Agent': random_useragent(allow_random_useragent),
    'X_FORWARDED_FOR': random_x_forward_for(allow_random_x_forward),
    'Referer': 'http://www.baidu.com',
    'Cookie': "",
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
# domians.py
# coding=utf-8
import os
import sys
import argparse
from utils.Crt import Crt
from utils.ilink import ILink
from common import save_result, read_json
from utils.brutedns import BruteDns

def run(args):
    domain = args.domain
    if not domain:
        print "Usage:domains,py -d test.com"
        sys.exit(1)
    outfile = '{0}.log'.format(domain)

    script_path = os.path.dirname(os.path.abspath(__file__))
    _cache_path = os.path.join(script_path, 'result/{0}'.format(domain))
    if not os.path.exists(_cache_path):
        os.makedirs(_cache_path, 0777)

    print '[*]Starting CRT fetch........'
    result = Crt(domain=domain).run()
    _cache_file = os.path.join(_cache_path, 'crt.json')
    save_result(_cache_file, result)
    print '\t [-]Fetch Complete | Found {}'.format(len(result))

    print '[*]Starting ILink fetch........'
    result = ILink(domain=domain).run()
    _cache_file = os.path.join(_cache_path, 'ilink.json')
    save_result(_cache_file, result)
    print '\t [-]Fetch Complete | Found {}'.format(len(result))

    print '[*]Starting Brute........'
    result = BruteDns(domain=domain).run()
    _cache_file = os.path.join(_cache_path, 'brute.json')
    save_result(_cache_file, result)
    print '\n\t [-]Fetch Complete | Found {}'.format(len(result))

    _cache_files = ['crt.json', 'ilink.json', 'brute.json']

    subdomains = []

    for file in _cache_files:
        _cache_file = os.path.join(_cache_path, file)
        json_data = read_json(_cache_file)
        if json_data:
            subdomains.extend(json_data)

    subdomains = list(set(subdomains))

    _result_file = os.path.join(script_path, outfile)
    save_result(_result_file, subdomains)

    print '\n[*]{0} {1} subdomains \n\tsave to {2}'.format(domain, len(subdomains), _result_file)


if __name__ == '__main__':
    parser = argparse.ArgumentParser(description="Domain Scan Ver:1.0")
    parser.add_argument("-d", "--domain", metavar="", help="domain name")
    args = parser.parse_args()

    try:
        run(args)
    except KeyboardInterrupt:
        sys.exit(1)

展开全文 >>

webC段扫描 

1
2
3
4
Web_C_Scan.py
run.sh
nmap.xml
result.xml
1
2
3
4
5
6
7
8
#!/bin/bash
if [ $# -lt 1 ];then
    echo "Example: $0 192.168.1.1/24"
    exit 1
fi
nmap -sn -PE --min-hostgroup 1024 --min-parallelism 1024 -oX nmap.xml $1

python Web_C_Scan.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
# coding=utf-8

import requests
from Queue import Queue
import time
import threading
import re

headers = {
    'user-agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36'}


class C_Scan(threading.Thread):
    def __init__(self, queue):
        threading.Thread.__init__(self)
        self._queue = queue

    def run(self):
        while not self._queue.empty():
            url = self._queue.get()
            print url
            try:
                r = requests.get(url=url, headers=headers, timeout=6)
                if r.status_code == 200:
                    print '[*]Web Service Found!%s' % url
                    try:
                        title = re.findall('<title>(.*?)</title>', r.text)[0]
                    except:
                        title = 'None'
                    f = open('result.html', 'a+')
                    f.write('<a href="' + url + '" target="_blank">' + url + '</a>' + '\t\t' + title + '\n')
                    f.write('<br>')
                    f.close()
            except Exception, e:
                pass


def create(ips):
    queue = Queue()
    req_m = ['http://', ]
    ports = ['80', '8080', ]
    dirs = ['', '/phpmyadmin', '/temp', ]
    for h in req_m:
        for i in ips:
            for j in ports:
                for k in dirs:
                    queue.put(str(h) + str(i) + ':' + str(j) + str(k))
    return queue


def main(ips):
    f = open('result.html', 'w')
    f.close()

    queue = create(ips)
    threads = []
    thread_count = 10
    for i in range(thread_count):
        threads.append(C_Scan(queue))
    for i in threads:
        i.start()
    for i in threads:
        i.join()


if __name__ == '__main__':
    with open('nmap.xml') as f:
        ips = re.findall('<address addr="(.*?)" addrtype=', f.read())
        main(ips)

展开全文 >>

web目录扫描 

1
2
3
4
5
dics 目录
proxy_use.py
user_agent_use.py
webdirscan.py
result.html
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
# proxy_use.py
# coding=utf-8
import random

def get_proxy():
    proxy_list = [
        {'http': '61.135.217.7:80'},
        {'http': '61.135.155.82:443'},
        {'http': '115.46.69.56:8123'},
        {'https': '106.15.42.179:33543'},
        {'https': '171.35.50.80:8118'},
        {'https': '121.228.162.101:8118'},
        {'https': '115.225.60.111:8118'},
        {'https': '202.104.113.35:53281'},
        {'https': '139.129.207.72:808'},
        {'https': '182.111.64.7:41766'},
        {'https': '219.234.5.128:3128'},

    ]
    return random.choice(proxy_list)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
# user_agent_use.py
# encoding: utf-8

import random


def get_user_agent():
    user_agent_list = [
        {
            'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36'},
        {
            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/535.20 (KHTML, like Gecko) Chrome/19.0.1036.7 Safari/535.20'}
    ]

    return random.choice(user_agent_list)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
# webdirscan.py
# coding=utf-8

import requests
from libs.output import *
from libs.utils.FileUtils import FileUtils
from libs.checkWAF import checkWaf
from Queue import Queue
import sys
import threading
import user_agent_use
import proxy_use
from optparse import OptionParser


class WebDirScanMain:
    def __init__(self, options):
        self.url = options.url
        self.ext = options.ext
        self.count = options.count

    class WebDirScan(threading.Thread):
        def __init__(self, queue, total):
            threading.Thread.__init__(self)
            self._queue = queue
            self._total = total

        def run(self):
            while not self._queue.empty():
                url = self._queue.get()
                threading.Thread(target=self.msg).start()
                try:
                    r = requests.get(url=url, headers=user_agent_use.get_user_agent(), timeout=8,
                                     proxies=proxy_use.get_proxy(), allow_redirects=True)
                    # checkWaf(url=url, headers=user_agent_use.get_user_agent(), timeout=8,
                    #          proxies=proxy_use.get_proxy(), allow_redirects=True)
                    if r.status_code == 200:
                        sys.stdout.write('\r' + '[*]%s\t\t\n' % (url))
                        f = open('result.html', 'a+')
                        f.write('<a href="' + url + '" target="_black">' + url + '</a>')
                        f.write('\r\n</br>')
                        f.close()
                except Exception, e:
                    pass

        def msg(self):
            per = 100 - float(self._queue.qsize()) / float(self._total) * 100
            # print self._total, self._queue.qsize()
            msg = '%s Left|%s All|Scan in %1.f%s' % (self._queue.qsize(), self._total, per, '%')
            sys.stdout.write('\r' + '[#]' + msg)
            sys.stdout.flush()

    def start(self):
        f = open('result.html', 'w')
        f.close()
        queue = Queue()

        f = open('./dics/%s.txt' % self.ext, 'r')
        for i in f:
            queue.put(self.url + i.rstrip('\n'))

        total = queue.qsize()

        threads = []
        thread_count = self.count
        for i in range(thread_count):
            threads.append(self.WebDirScan(queue, total))
        for i in threads:
            i.start()
        for i in threads:
            i.join()


if __name__ == '__main__':
    print '''
__        __   _     ____  _      ____                  
\ \      / /__| |__ |  _ \(_)_ __/ ___|  ___ __ _ _ __  
 \ \ /\ / / _ \ '_ \| | | | | '__\___ \ / __/ _` | '_ \ 
  \ V  V /  __/ |_) | |_| | | |   ___) | (_| (_| | | | |
   \_/\_/ \___|_.__/|____/|_|_|  |____/ \___\__,_|_| |_|
                                                                
    '''
    print '''
    Design By:ShowMeShell   Version:1.0   
    '''
    parser = OptionParser()
    parser = OptionParser()
    parser.add_option("-u", "--url", dest="url", help="Target url for scan")
    parser.add_option("-f", "--file", dest="ext", help="Target url ext")
    parser.add_option("-t", "--thread", dest="count", type="int", default=10, help="Scan Thread_count")

    (options, args) = parser.parse_args()
    if options.url and options.ext:
        d = WebDirScanMain(options)
        d.start()
        sys.exit(1)
    else:
        parser.print_help()
        sys.exit(1)

展开全文 >>

创造高度可定制字典 

1
2
3
rule.ini
pass_o.txt
dictonary.txt
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
# coding=utf-8

import exrex
import sys

'''
host=http://123.456.com or 123.456.com or https://123.456.com or  http://123.456.com/
'''

web_white = ['com', 'cn', 'gov', 'edu', 'org', 'www']


def host_para(host):
    if '://' in host:
        host = host.split('://')[1].replace('/', '')

    if '/' in host:
        host = host.replace('/', '')
    return host


# 结合常用字典创造新的dic
def dic_creat(hosts):
    web_dics = hosts.split('.')
    # 核心的生成规则，写入配置文件

    f_rule = open('rule.ini', 'r')
    for i in f_rule:
        if '#' != i[0]:
            rule = i

    f_pass_out = open('dictonary.txt', 'w')
    f_pass_out.close()

    for web_dic in web_dics:
        if web_dic not in web_white:
            f_pass = open('pass_0.txt', 'r')
            for dic_pass in f_pass:
                dics = list(exrex.generate(rule.format(web_dic=web_dic, dic_pass=dic_pass.strip('\n'))))

                for dic in dics:
                    if len(dic) > 4:
                        f_pass_out = open('dictonary.txt', 'a+')
                        f_pass_out.write(dic + '\n')
                        f_pass_out.close()
                        print dic


if __name__ == '__main__':
    if len(sys.argv) == 2:
        dic_creat(host_para(sys.argv[1]))
    else:
        print 'Usage:%s www.demo.com' % sys.argv[0]
        sys.exit(-1)

rule.ini 配置文件

1
2
3
# 字典的生成程序的配置文件
# 配置规则形如：{web_dic}[!@#]{dic_pass} web_dic为host中可能的字典元素，dic_pass为密码母本的内容
(|{web_dic})(|!|@|#)(|{dic_pass})(|!|@|#)(|201[4567890])

pass_0.txt

1
2
3
admin
password
123456

展开全文 >>

Zoomeye信息采集

实例5-zoomeye信息采集

zoomeys

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
# coding=utf-8
import requests
import json
import sys


# https://www.zoomeye.org/doc#host-search
def login():
    url_login = 'https://api.zoomeye.org/user/login'
    data = {
        "username": "zoomeye账号",
        "password": "zoomeys密码"
    }
    data = json.dumps(data)

    r = requests.post(url=url_login, data=data)
    return json.loads(r.content)['access_token']


def main(keyword):
    url = 'https://api.zoomeye.org/host/search?query=%s' % (keyword)
    headers = {'Authorization': 'JWT ' + login()}
    r = requests.get(url=url, headers=headers)
    # print r.content
    datas = json.loads(r.content)['matches']
    for data in datas:
        print data['ip'], data['portinfo']['port']


if __name__ == '__main__':
    if len(sys.argv) != 2:
        print 'Enter:%s keyword' % sys.argv[0]
        sys.exit(-1)
    else:
        for i in (1, 11):
            input = str(sys.argv[1] + '&page=' + str(i))
            main(input)

展开全文 >>

代理IP采集

实例4-代理ip采集

代理ip

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
# coding=utf-8
import requests
from bs4 import BeautifulSoup as bs
import re


def proxy_spider():
    headers = {
        'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36'
    }
    url = 'http://www.xicidaili.com/nn'
    r = requests.get(url=url, headers=headers)
    # print r.status_code, r.content
    soup = bs(r.content, 'lxml')
    datas = soup.find_all(name='tr', attrs={'class': re.compile('|[^odd]')})
    for data in datas:
        soup_proxy_content = bs(str(data), 'lxml')
        soup_proxy = soup_proxy_content.find_all(name='td')
        # print soup_proxy
        ip = str(soup_proxy[1].string)
        port = str(soup_proxy[2].string)
        types = str(soup_proxy[5].string)
        proxy_check(ip, port, types)


def proxy_check(ip, port, types):
    proxy = {}
    proxy[types.lower()] = '%s:%s' % (ip, port)
    try:
        r = requests.get('http://showmeshell.top/', proxies=proxy, timeout=6)
        if r.status_code == 200:
            print proxy
    except Exception, e:
        pass

proxy_spider()

展开全文 >>

百度URL采集

实例3-百度url采集

百度url

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
# coding=utf-8
import requests
import threading
import Queue
from bs4 import BeautifulSoup as bs
import time
import re
import sys

# https://www.baidu.com/s?wd=keyword&pn=10
# wd控制keyword，pn控制页码（0-750）

headers = {
    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36'
}


class BaiduSpider(threading.Thread):
    def __init__(self, queue):
        threading.Thread.__init__(self)
        self._queue = queue

    def run(self):
        while not self._queue.empty():
            url = self._queue.get()
            try:
                self.spider(url)
            except Exception, e:
                print e
                pass

    def spider(self, url):
        r = requests.get(url=url, headers=headers)
        print r.status_code
        soup = bs(r.content, 'lxml')
        urls = soup.find_all(name='a', attrs={'data-click': re.compile(('')), 'class': None})
        for url in urls:
            r_get_url = requests.get(url=url['href'], headers=headers, timeout=8)
            if r_get_url.status_code == 200:
                print r_get_url.url
                url_para = r_get_url.url
                url_index_tmp = url_para.split('/')
                url_index = url_index_tmp[0] + '//' + url_index_tmp[2]
                print url_index
                f1 = open('out_para.txt', 'a+')
                f1.write(url_para + '\n')
                f1.close()
                with open('out_index.txt') as f:
                    if url_index not in f.read():
                        f2 = open('out_index.txt', 'a+')
                        f2.write(url_index + '\n')
                        f2.close()


def main(keyword):
    queue = Queue.Queue()
    for i in range(0, 760, 10):
        queue.put('https://www.baidu.com/s?wd=%s&pn=%s' % (keyword, str(i)))

    threads = []
    threading_count = 3
    for i in range(threading_count):
        threads.append(BaiduSpider(queue))
    for t in threads:
        t.start()
    for t in threads:
        t.join()


if __name__ == '__main__':
    f1 = open('out_para.txt', 'w')
    f1.close()
    f2 = open('out_index.txt', 'w')
    f2.close()
    if len(sys.argv) != 2:
        print 'Enter:%s keyword' % sys.argv[0]
        sys.exit(-1)
    else:
        main(sys.argv[1])

展开全文 >>

seebug爬取POC

实例2-seebug爬取poc

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
# coding=utf-8
import requests
import threading
import Queue
from bs4 import BeautifulSoup as bs
import json
import time

'''
page=https://www.seebug.org/vuldb/vulnerabilities?page=1
vul=https://www.seebug.org/vuldb/ssvid-15334
exchange=https://www.seebug.org/vuldb/exchange/15334 POST{"type":"poc","anonymous":true}
down=https://www.seebug.org/vuldb/downloadPoc/15334
'''
# url = 'https://www.seebug.org/vuldb/vulnerabilities?page=1'
headers = {
    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36',
    'Host': 'www.seebug.org',
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
    'Accept-Encoding': 'gzip, deflate, br',
    'Accept-Language': 'zh-CN,zh;q=0.9',
    'Referer': 'https://www.baidu.com/',
    'X-CSRFToken': 'dPgYO696wm9XtQcGKwxeuKveLtUV1oQe',
    'X-Requested-With': 'XMLHttpRequest',
    'Connection': 'keep-alive',
    'Cookie': '__jsluid=b7c21a10da9eeec93af0dc370f80ee4a; csrftoken=dPgYO696wm9XtQcGKwxeuKveLtUV1oQe; sessionid=enhf0bkv90q0k8h15bm5fi2a3kr79345; Hm_lvt_6b15558d6e6f640af728f65c4a5bf687=1543295016,1544516306,1544516384,1544516393; messages="35eb9b85037593abe32d476bd15d22ecf14483df$[[\"__json_message\"\0540\05425\054\"Login succeeded. Welcome\054 966371b7.\"]\054[\"__json_message\"\0540\05425\054\"You are logged in as 966371b7.\"]]"; __jsl_clearance=1544599069.178|0|G6429BqT7Ja8gjPOWljYezPMPkw%3D; Hm_lpvt_6b15558d6e6f640af728f65c4a5bf687=1544600943',
    'Connection': 'keep-alive',
}


class SeebugPoc(threading.Thread):
    def __init__(self, queue):
        threading.Thread.__init__(self)
        self._queue = queue

    def run(self):
        while not self._queue.empty():
            url = self._queue.get_nowait()
            spider(url)


def spider(url):
    dic = {}
    dic['type'] = 'poc'
    dic['anonymous'] = 'true'
    datas = json.dumps(dic)
    r = requests.get(url=url, headers=headers)
    print r.status_code
    soup = bs(r.content, 'lxml')
    vulids = soup.find_all(name='a', attrs={'class': 'vul-title'})
    for vulid in vulids:
        print vulid['href'].split('-')[-1], vulid['title']
        down_url = 'https://www.seebug.org/vuldb/downloadPoc/' + vulid['href'].split('-')[-1]
        print down_url
        down_r = requests.get(url=down_url, headers=headers)
        if down_r.status_code == 200:
            f = open('seebugPoc/' + vulid['href'].split('-')[-1], 'w')
            f.write(down_r.content)
            f.close
        else:
            print 'No Poc'
        time.sleep(2)


def main():
    queue = Queue.Queue()
    for i in range(2545, 2550):
        queue.put('https://www.seebug.org/vuldb/vulnerabilities?page=' + str(i))

    threads = []
    threading_count = 3
    for i in range(threading_count):
        threads.append(SeebugPoc(queue))
    for t in threads:
        t.start()
    for t in threads:
        t.join()


if __name__ == '__main__':
    main()

展开全文 >>

爬虫三

BeautifulSoup

  • 解析内容：
    • from bs4 import BeautifulSoup
    • soup = BeautifulSoup(html_doc)
  • 浏览数据
    • soup.title
    • soup.title.string
  • BeautifulSoup正则
    • soup.find_all(name=’x’,attrs={‘xx’:re.compile(‘xxx’)})
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
# coding=utf-8
import requests
from bs4 import BeautifulSoup
import re

url = 'http://www.ysdyzx.com'
headers = {
    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36'
}
r = requests.get(url=url, headers=headers)
soup = BeautifulSoup(r.content, 'lxml')
news=soup.find_all(name='a',attrs={'href':re.compile('/a/xiaoyuandongtai/xinwenkuaixun/\d*?.html')})
# news = soup.find_all(name='a', attrs={'target': '_blank'})
#
for new in news:
    print new.string

hackhttp

1
2
3
4
5
# 发起get，post请求，发起http原始数据包
import hackhttp
hh = hackhttp.hackhttp()
url = "http://xxxxx"
code,head,html,redirect_url,log = hh.http(url)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
import hackhttp
import re
from bs4 import BeautifulSoup as BS

def tomcat(raw):
    url = 'http://www.cnvd.org.cn/flaw/list.htm?flag=true'
    hh = hackhttp.hackhttp()
    code, head, html, redirect, log = hh.http(url=url, raw=raw)
    soup = BS(html, 'lxml')
    tomcat_html = soup.tbody
    tomcat_cnvds = BS(str(tomcat_html), 'lxml')
    cnvds = tomcat_cnvds.find_all(name='a', attrs={'href': re.compile('/flaw/show/CNVD-.*?')})
    for cnvd in cnvds:
        print cnvd['title']

raw_start = '''
POST /flaw/list.htm?flag=true HTTP/1.1
Host: www.cnvd.org.cn
Content-Length: 388
Cache-Control: max-age=0
Origin: http://www.cnvd.org.cn
Upgrade-Insecure-Requests: 1
Content-Type: application/x-www-form-urlencoded
User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36
Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8
Referer: http://www.cnvd.org.cn/flaw/list.htm?flag=true
Accept-Encoding: gzip, deflate
Accept-Language: zh-CN,zh;q=0.9
Cookie: __jsluid=558a536188ba647ef4c0e8bfc29ad880; bdshare_firstime=1537405268263; JSESSIONID=9E0ED054CFE20F5D376D50D5BF537B06
number=%E8%AF%B7%E8%BE%93%E5%85%A5%E7%B2%BE%E7%A1%AE%E7%BC%96%E5%8F%B7&startDate=&endDate=&field=&order=&flag=true&keyword=tomcat&condition=1&keywordFlag=0&cnvdId=&cnvdIdFlag=0&baseinfoBeanbeginTime=&baseinfoBeanendTime=&baseinfoBeanFlag=0&refenceInfo=&referenceScope=-1&manufacturerId=-1&categoryId=-1&editionId=-1&causeIdStr=&threadIdStr=&serverityIdStr=&positionIdStr=&max=20&offset='''

for pages_count in range(0, 180, 20):
    raw = raw_start + str(pages_count)
    tomcat(raw)

实例1-ichunqiu及jiandan

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
# 多线程实例
# coding=utf-8
import requests
import json
import thread
import time


def spider(data):
    headers = {'User-agent': 'showmeshell spider for ichunqiu ver=1.0'}
    url = 'http://www.ichunqiu.com/courses/ajaxCourses'
    r = requests.post(url=url, headers=headers, data=data)
    datas = json.loads(r.content)

    for data in datas['course']['result']:
        print data['courseName'], data['producerName']


for i in range(1, 10):
    data = 'courseTag=&courseDiffcuty=&IsExp=&producerId=&orderField=&orderDirection=&tagType=2&pageIndex=' + str(i)
    thread.start_new_thread(spider, (data,))
    time.sleep(0.1)
#################

# coding=utf-8
import requests
import threading
import Queue


class JianDan(threading.Thread):
    def __init__(self, queue):
        threading.Thread.__init__(self)
        self._queue = queue

    def run(self):
        while not self._queue.empty():
            url = self._queue.get_nowait()
            self.spider(url)

    def spider(self, url):
        headers = {'User-agent': 'showmeshell spider for jiandan ver=1.0'}
        r = requests.get(url=url, headers=headers)
        print r.status_code, len(r.content)


def main():
    queue = Queue.Queue()
    for i in range(2180, 2184):
        queue.put('http://jandan.net/ooxx/page-' + str(i))

    threads = []
    thread_count = 3

    for i in range(thread_count):
        threads.append(JianDan(queue))
    for t in threads:
        t.start()
    for t in threads:
        t.join()


if __name__ == '__main__':
    main()
#########
# coding=utf-8

import requests
from bs4 import BeautifulSoup as bs
import threading
import Queue
import urllib

url = 'http://jandan.net/pic/page-2162'


class Jiandan(threading.Thread):
    def __init__(self, queue):
        threading.Thread.__init__(self)
        self._queue = queue

    def run(self):
        while not self._queue.empty():
            url = self._queue.get_nowait()
            self.spider(url)

    def spider(self, url):
        headers = {'User-agent': 'jiandan spider by showmeshell ver=1.0'}
        r = requests.get(url=url, headers=headers)
        soup = bs(r.content, 'lxml')
        imgs = soup.find_all(name='img', attrs={})
        for img in imgs:
            if 'onload' in str(img):
                img = img['org_src']
            else:
                img = img['src']
            name = img.split('/')[-1]
            urllib.urlretrieve(img, filename='img/' + name)


def main():
    queue = Queue.Queue()
    for i in range(2159, 2162):
        queue.put('http://jandan.net/pic/page-2162' + str(i))

    threads = []
    threading_count = 3
    for i in range(threading_count):
        threads.append(Jiandan(queue))
    for i in threads:
        i.start()
    for i in threads:
        i.join()


if __name__ == '__main__':
    main()

展开全文 >>

基础知识

广度与深度

seebug/TangScan/BugScna平台
sqlmap/mitmproxy/mitmdump

python路线

python正则表达式/web编程/多线程编程/网络编程/数据库编程

能力

  • 目录扫描
    web+多线程{requests+threading+Queue:[后台|敏感文件(svn|upload)|敏感目录(phpmyadmin)]}
  • 信息搜集
    web+数据库{中间件(Tomcat|Jboss)+C段web信息+搜集特定程序}
  • 信息匹配&SQL注入
    web+正则{抓取信息(用户名|邮箱?)+SQL注入}
  • 反弹shell
    网络

python正则表达式

1
2
3
4
5
6
7
8
>>> import re
>>> pattern = re.compile('hello')
>>> match = pattern.match('hello word!')
>>> print match.group()
hello
>>> word = re.findall('hello','hello word!')
>>> word
['hello']

python通过re模块提供对正则表达式的支持
1.先将正则表达式的字符串形式编译为pattern实例
2.使用Pattern实例处理文本并获得匹配结果；（match/find/search）
3.使用实例获得信息，进行其他的操作。

python正则表达式

正则

python-Web

  • urllib
    • urllib.urlopen()
    • urllib.urlretrieve()
  • urllib2
    • urllib2.urlopen()
    • urllib2.Requests()
  • requests模块
    • 发送网络请求
      • get/post/put/delete/head/options
    • 为URL传递参数
      • payload = {‘key1’:’value1’,’key2’:’value2’}
      • r=requests.get(“http://xx/get",params=payload)
      • print(r.url)
      • http://xx/get?key2=value2&key1=value1
    • 响应内容
      • r=request.get(url)
      • r.text
      • r.encoding ‘utf-8’
      • r.encoding=’ISO-8859-1’
    • 二进制响应内容
      • r=request.get(url)
      • r.content
    • 定制请求头
      • url=’http://xx/get'
      • headers={‘content-type’:’application/json’}
      • r=requests.get(url,headers=headers)
    • 复杂的post请求
      • payload={‘key1’:’value1’,’key2’;’value2’}
      • r=requests.post(“http:xxx/post0”,data=payload)
    • 响应状态码
      • r=requests.get(‘http:xxx/xx’)
      • r.status_code
    • 响应头
      • r.headers
    • Cookies
      • r.cookies
      • r.cookies[‘cookie_name’]
    • 超时
      • requests.get(‘http:xxx’,timeout=0.001)
    • 错误和异常
      • 遇到网络问题，如DNS查询失败，拒绝连接时，Requests会抛出一个connecterror异常，遇到罕见的无效http响应时，requests则会抛出HttpError异常。若请求超时，则抛出一个Timeout异常。
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
# coding =utf-8
import requests
import re

url = 'http://xxxxx'


def lesson(url):
    r = requests.get(url)
    url_txt = r.text
    word = re.findall(r"target='_blank' title='(.*?)'>", url_txt)
    for i in word:
        print i

for i in range(0,10):
    url_end=url+str(i)+'.htm'
    lesson(url_end)

python多线程

  • 进程

    进程是程序一次执行，每个进程都有自己的地址空间，内存，数据栈及其它记录其行为轨迹的辅助数据。

  • 线程

    有的线程运行在同一个进程当中，共享相同的运行环境，线程有开始顺序执行和结束三个部分。

thread模块

  • start_new_thread(function,args kwargs=None)

    产生一个新的线程，在新的线程中用指定的参数和可选的kwargs来调用这个函数，在使用这种方法时，一定要加time.sleep()，否则每个线程都可能不执行。还有一个缺点，遇到复杂问题时，线程数不易控制。

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
# 扫描C段
# coding =utf-8
import thread
import time
from subprocess import Popen, PIPE

def ping_check(ip):
    check = Popen(['/bin/bash', '-c', 'ping -c 2 ' + ip], stdin=PIPE, stdout=PIPE)
    data = check.stdout.read()
    if 'ttl' in data:
        print '%s is UP' % ip

def main():
    for i in range(1, 255):
        ip = '180.97.33.' + str(i)
        thread.start_new_thread(ping_check, (ip,))
        time.sleep(0.1)

if __name__ == '__main__':
    main()

  • thread类

    • 使用threading模块
    • 子类化Thread类
    • 解决了线程可控的问题
      1
      2
      3
      4
      5
      6
      7
      8
      9
      10
      11
      12
      13
      14
      15
      16
      17
      18
      19
      20
      21
      22
      23
      24
      25
      # coding =utf-8
      import threading
      import time
      import requests
      import sys

      def fun1():
          time_start = time.time()
          r = requests.get(url="http://www.baidu.com")
          times = time.time() - time_start
          sys.stdout.write('Status:%s---%s---%s\n' % (r.status_code, times, time.strftime('%H:%M:%s')))

      def main():
          threads = []
          threads_count = 10
          for i in range(threads_count):
              t = threading.Thread(target=fun1, args=())
              threads.append(t)
          for i in range(threads_count):
              threads[i].start()
          for i in range(threads_count):
              threads[i].join()

      if __name__ == '__main__':
          main()

  • 生产者-消费者问题和Queue模块

    • Queue模块（qsize(),empty(),full(),put(),get())
    • 完美搭档，Queue+Thread
    • 解决了生产参数和计算结果时间都不确定的问题。
      1
      2
      3
      4
      5
      6
      7
      8
      9
      10
      11
      12
      13
      14
      15
      16
      17
      18
      19
      20
      21
      22
      23
      24
      25
      26
      27
      28
      29
      30
      31
      32
      33
      34
      # coding =utf-8
      import threading
      import Queue
      import sys
      from subprocess import PIPE, Popen

      class DoRun(threading.Thread):
          def __init__(self, queue):
              threading.Thread.__init__(self)
              self._queue = queue

          def run(self):
              while not self._queue.empty():
                  ip = self._queue.get()
                  check_ping = Popen(['/bin/bash', '-c', 'ping -c 2 ' + ip], stdin=PIPE, stdout=PIPE)
                  data = check_ping.stdout.read()
                  if 'ttl' in data:
                      sys.stdout.write(ip + ' is UP\n')

      def main():
          threads = []
          threads_count = 10
          queue = Queue.Queue()
          for i in range(1, 255):
              queue.put('192.30.252.' + str(i))
          for i in range(threads_count):
              threads.append(DoRun(queue))
          for i in threads:
              i.start()
          for i in threads:
              i.join()

      if __name__ == '__main__':
          main()

python网络编程

  • 套接字是一种具有通信端点概念的计算机网络数据结构，应用程序开始通信之前必须建立套接字。
  • python支持：AF_UNIX,AF_NETLINK,AF_INET，其中AF_INET基于网络的套接字。
  • socket模块
    • socket模块的socket()函数用来创建套接字。
    • socket(socket_family,socket_type,protocol=0)
    • 创建一个TCP/IP套接字，tcpsock=socket(socket.AF_INET,socket.SOCK_STREAM)
服务器端套接字函数
s.bind() 绑定地址(主机，端口)到套接字
s.listen() 开始TCP监听
s.accept() 被动接受TCP客户端连接，(阻塞式)等待连接的到来。
客户端套接字函数
s.connect() 主动初始化TCP服务器连接
s.connect_ex() Connect函数的扩展版本，出错时返回错误代码而不是掏出异常
公共用途套接字函数
s.recv() 接受TCP数据
s.send() 发送TCP数据
s.sendall() 完整发送TCP数据
s.recvfrom() 接受UDP数据
s.sendto() 发送UDP数据
s.getpeemame() 连接到当前套接字的远端地址
s.getsockname() 当前套接字的地址
s.getsockopt() 返回指定套接字的参数
s.setsockopt() 设置指定套接字的参数
s.close() 关闭套接字 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
#创建一个TCP服务器
ss=socket()				#创建服务器套接字
ss.bind()				#把地址绑定到套接字
ss.listen()				#监听连接
inf_loop:				#服务器无限循环
    cs=ss.accept()		#接受客户端连接
cmmon_loop:				#通信循环
    cs.recv()/cs.send()	#对话（接受与发送）
cs.close()    			#关闭客户端套接字
#创建一个TCP客户端
cs.socket()				#创建客户端套接字
cs.connect()			#尝试连接套接字
common_loop:			#通信循环
    cs.recv()/cs.send() #对话（接受与发送）
cs.close()    			#关闭客户端套接字
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
from socket import *
from time import ctime
from subprocess import PIPE,Popen
HOST = ''
PROT = 23333
BUFSIZE = 1024

ADDR = (HOST, PROT)

tcpServer = socket(AF_INET, SOCK_STREAM)
tcpServer.bind(ADDR)
tcpServer.listen(5)

while True:
    print 'waiting for connection....'
    tcpClient, addr = tcpServer.accept()
    print '..connected from:', addr
    while True:
        data = tcpClient.recv(BUFSIZE)
        if not data:
            break
        cmd=Popen(['/bin/bash','-c',data],stdin=PIPE,stdout=PIPE)
        data=cmd.stdout.read()
        tcpClient.send('[%s] %s' % (ctime(), data))

tcpClient.close()
tcpServer.close()
#**************************************************#
from socket import *

HOST = 'xx.xx.xx.xx'
PROT = 23333
BUFSIZE = 1024

ADDR = (HOST, PROT)

tcpClient = socket(AF_INET, SOCK_STREAM)
tcpClient.connect(ADDR)
while True:
    data = raw_input('~:')
    if not data:
        break
    tcpClient.send(data)
    data = tcpClient.recv(BUFSIZE)
    if not data:
        break
    print data
    
tcpClient.close()

python数据库编程

  • python DB API包含内容
    • 数据库连接对象connection
    • 数据库交互对象cursor
    • 数据库异常类exceptions
    • 过程
      • 开始->创建connection->获取cursor->执行查询，执行命令，获取数据，处理数据->关闭cursor->关闭connection->结束

数据库连接对象connection

连接对象：建立python客户端与数据库的网络连接

创建方法：MySQLdb.Connect（参数）

参数名 类型 说明
host 字符串 mysql服务器地址
port 数字 mysql服务器端口号
user 字符串 用户名
passwd 字符串 密码
db 字符串 数据库名
charset 字符串 连接编码

connection对象支持的方法：

方法名 说明
cursor() 使用该连接创建并返回游标
commit() 提交当前事物
rollback() 回滚当前事物
close() 关闭连接

cursor对象支持的方法：

参数名 说明
execute(op,[args]) 执行一个数据库查询命令（select，insert，delete）
fetchone() 取得结果集里的下一行
fetchmany(size) 获得结果集里的下几行
fetchall() 获得结果集剩下的所有行
rowcount 最后一次execute返回数据的行数或影响行数
close() 关闭 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
import MySQLdb

conn = MySQLdb.connect(
    host='xx.xx.xx.xx',
    port=123,
    user='root',
    passwd='xxxx',
)
cus = conn.cursor()
sql = 'select version()'
cus.execute(sql)

print cus.fetchone()

cus.close()
conn.close()

展开全文 >>

tag:

    缺失模块。
    1、请确保node版本大于6.2
    2、在博客根目录（注意不是yilia根目录）执行以下命令：
    npm i hexo-generator-json-content --save

    3、在根目录_config.yml里添加配置： 

      jsonContent:
    meta: false
    pages: false
    posts:
      title: true
      date: true
      path: true
      text: false
      raw: false
      content: false
      slug: false
      updated: false
      comments: false
      link: false
      permalink: false
      excerpt: false
      categories: false
      tags: true

<br><br>城南花已开<br>