powerdlz23 1.2.2 → 1.2.4

package/pto/crypto/rc4/csharp/rc4.cs

@@ -0,0 +1,119 @@
+using System;
+using System.IO;
+
+namespace Funny.Crypto
+{
+	public class RC4Cipher
+	{
+		private uint[] s = new uint[256];
+		private byte i, j;
+
+		public RC4Cipher(byte[] key) {
+			int k = key.Length;
+			if (k < 1 || k > 256) {
+				throw new RC4KeySizeException(k);
+			}
+
+			for (uint i = 0; i < 256; i++) {
+				s[i] = i;
+			}
+
+			byte j = 0;
+			uint t = 0;
+			for (int i = 0; i < 256; i++) {
+				j = (byte)(j + s[i] + key[i % k]);
+				t = s[i];
+				s[i] = s[j];
+				s[j] = t;
+			}
+		}
+
+		public void XORKeyStream(byte[] dst, int dstOffset, byte[] src, int srcOffset, int count) {
+			if (count == 0)
+				return;
+
+			byte i = this.i;
+			byte j = this.j;
+			uint t = 0;
+			for (int k = 0; k < count; k ++) {
+				i += 1;
+				j = (byte)(s[i] + j);
+				t = s[i];
+				s[i] = s[j];
+				s[j] = t;
+				dst[k + dstOffset] = (byte)(src[k + srcOffset] ^ (byte)(s[(byte)(s[i] + s[j])]));
+			}
+			this.i = i;
+			this.j = j;
+		}
+	}
+
+	public class RC4Stream : Stream
+	{
+		private Stream stream;
+		private RC4Cipher cipher;
+
+		public RC4Stream(Stream stream, byte[] key) {
+			this.stream = stream;
+			this.cipher = new RC4Cipher(key);
+		}
+
+		public override int Read(byte[] buffer, int offset, int count) {
+			count = stream.Read(buffer, offset, count);
+			cipher.XORKeyStream(buffer, offset, buffer, offset, count);
+			return count;
+		}
+
+		public override void Write(byte[] buffer, int offset, int count) {
+			byte[] dst = new byte[count];
+			cipher.XORKeyStream(dst, 0, buffer, offset, count);
+			stream.Write(dst, 0, count);
+		}
+
+		public override bool CanRead {
+			get { return stream.CanRead; }
+		}
+
+		public override bool CanSeek {
+			get { return stream.CanSeek; }
+		}
+
+		public override bool CanWrite {
+			get { return stream.CanWrite; }
+		}
+
+		public override long Length {
+			get { return stream.Length; }
+		}
+
+		public override long Position {
+			get { return stream.Position; }
+			set { stream.Position = value; }
+		}
+
+		public override long Seek(long offset, SeekOrigin origin) {
+			return stream.Seek(offset, origin);
+		}
+
+		public override void SetLength(long length) {
+			stream.SetLength(length);
+		}
+
+		public override void Flush() {
+			stream.Flush();
+		}
+	}
+
+	public class RC4KeySizeException : Exception
+	{
+		private int size;
+
+		public RC4KeySizeException(int size) {
+			this.size = size;
+		}
+
+		public override string Message {
+			get { return "RC4Stream: invalid key size " + size; }
+		}
+	}
+}

package/pto/crypto/rc4/csharp/rc4_echo_client.cs

@@ -0,0 +1,78 @@
+using System;
+using System.IO;
+using System.Net.Sockets;
+using Funny.Crypto;
+
+// mcs rc4_echo_client.cs rc4.cs ../../dh64/csharp/dh64.cs
+class MainClass
+{
+	private static DH64 dh64 = new DH64();
+	private static Random random = new Random();
+
+	public static void Main(string[] args) {
+		TcpClient conn = new TcpClient("127.0.0.1", 10010);
+		Console.WriteLine("client connect");
+
+		Stream stream = conn.GetStream();
+		BinaryReader reader = new BinaryReader(stream);
+		BinaryWriter writer = ConnInit(reader);
+
+		byte[] buffer = new byte[1024];
+		for (;;) {
+			int length = WriteRandomBytes(writer, buffer);
+			uint n = reader.ReadUInt32();
+			byte[] recv = reader.ReadBytes((int)n);
+			if (!ByteArrayEquals(buffer, recv, length)) {
+				Console.WriteLine("send != recv");
+				Console.WriteLine("send: {0}", BitConverter.ToString(buffer, length));
+				Console.WriteLine("recv: {0}", BitConverter.ToString(recv));
+				return;
+			}
+		}
+	}
+
+	// Do DH64 key exchange and return a RC4 writer
+	private static BinaryWriter ConnInit(BinaryReader r) {
+		ulong privateKey;
+		ulong publicKey;
+		dh64.KeyPair(out privateKey, out publicKey);
+		Console.WriteLine("client public key: {0}", publicKey);
+
+		new BinaryWriter(r.BaseStream).Write(publicKey);
+		ulong srvPublicKey = r.ReadUInt64();
+		Console.WriteLine("server public key: {0}", srvPublicKey);
+
+		ulong secret = dh64.Secret(privateKey, srvPublicKey);
+		Console.WriteLine("secret: {0}", secret);
+
+		byte[] key;
+		using (MemoryStream ms = new MemoryStream()) {
+			new BinaryWriter(ms).Write(secret);
+			key = ms.ToArray();
+		}
+		Console.WriteLine("key: {0}", BitConverter.ToString(key).Replace("-", "").ToLower());
+
+		return new BinaryWriter(
+			new RC4Stream(r.BaseStream, key)
+		);
+	}
+
+	private static int WriteRandomBytes(BinaryWriter w, byte[] buffer) {
+		int length = random.Next(buffer.Length);
+		for (int i = 0; i < length; i ++) {
+			buffer[i] = (byte)random.Next(256);
+		}
+		w.Write((uint)length);
+		w.Write(buffer, 0, length);
+		return length;
+	}
+
+	private static bool ByteArrayEquals(byte[] a1, byte[] a2, int length) {
+		if (a2.Length != length)
+			return false;
+		for (int i=0; i<length; i++)
+			if (a1[i]!=a2[i])
+				return false;
+		return true;
+	}
+}

package/pto/crypto/rc4/go/rc4_echo_client.go

@@ -0,0 +1,102 @@
+package main
+
+import (
+	"bytes"
+	"crypto/cipher"
+	"crypto/rc4"
+	"encoding/hex"
+	"flag"
+	"github.com/funny/binary"
+	dh64 "github.com/funny/crypto/dh64/go"
+	"log"
+	"math/rand"
+	"net"
+	"time"
+)
+
+func main() {
+	addr := flag.String("addr", "127.0.0.1:10010", "server address")
+	flag.Parse()
+
+	conn, err := net.Dial("tcp", *addr)
+	if err != nil {
+		log.Print("connect failed: ", err)
+		return
+	}
+	log.Print("client connect")
+
+	writer, reader, err := conn_init(conn)
+	if err != nil {
+		log.Print("conn init failed: ", err)
+		return
+	}
+
+	send := make([]byte, 0, 1024)
+	for {
+		send = randomData(send)
+		writer.WritePacket(send, binary.SplitByUint32LE)
+		if writer.Error() != nil {
+			log.Print("send failed: ", writer.Error())
+			return
+		}
+		recv := reader.ReadPacket(binary.SplitByUint32LE)
+		if reader.Error() != nil {
+			log.Print("receive failed: ", reader.Error())
+			return
+		}
+		if !bytes.Equal(send, recv) {
+			log.Print("send != recv")
+			log.Print("send: ", hex.EncodeToString(send))
+			log.Print("recv: ", hex.EncodeToString(recv))
+			return
+		}
+	}
+}
+
+// Do DH64 key exchange and return a RC4 writer.
+func conn_init(conn net.Conn) (*binary.Writer, *binary.Reader, error) {
+	var (
+		writer = binary.NewWriter(conn)
+		reader = binary.NewReader(conn)
+	)
+
+	rand.Seed(time.Now().UnixNano())
+
+	privateKey, publicKey := dh64.KeyPair()
+	log.Print("client public key: ", publicKey)
+
+	writer.WriteUint64LE(publicKey)
+	if writer.Error() != nil {
+		return nil, nil, writer.Error()
+	}
+	serverPublicKey := reader.ReadUint64LE()
+	if reader.Error() != nil {
+		return nil, nil, reader.Error()
+	}
+	log.Print("server public key: ", serverPublicKey)
+
+	secert := dh64.Secret(privateKey, serverPublicKey)
+	log.Print("secert: ", secert)
+
+	key := make([]byte, 8)
+	binary.PutUint64LE(key, secert)
+	rc4stream, err := rc4.NewCipher(key)
+	if err != nil {
+		return nil, nil, err
+	}
+	log.Print("key: ", hex.EncodeToString(key))
+
+	writer = binary.NewWriter(cipher.StreamWriter{
+		W: conn,
+		S: rc4stream,
+	})
+	return writer, reader, nil
+}
+
+func randomData(b []byte) []byte {
+	a := b[:rand.Intn(cap(b))]
+	for i := 0; i < len(a); i++ {
+		a[i] = byte(rand.Int() % 256)
+	}
+	return a
+}

package/pto/crypto/rc4/go/rc4_echo_server.go

@@ -0,0 +1,110 @@
+package main
+
+import (
+	"crypto/cipher"
+	"crypto/rc4"
+	"encoding/hex"
+	"flag"
+	"github.com/funny/binary"
+	dh64 "github.com/funny/crypto/dh64/go"
+	"log"
+	"math/rand"
+	"net"
+	"time"
+)
+
+func main() {
+	addr := flag.String("addr", "127.0.0.1:10010", "server address")
+	flag.Parse()
+
+	lsn, err := net.Listen("tcp", *addr)
+	if err != nil {
+		log.Print("listen failed: ", err)
+		return
+	}
+	log.Print("server wait")
+
+	for {
+		conn, err := lsn.Accept()
+		if err != nil {
+			log.Print("accept failed: ", err)
+			return
+		}
+		log.Print("server accpet: ", conn.RemoteAddr())
+
+		go func() {
+			writer, reader, err := conn_init(conn)
+			if err != nil {
+				log.Print("conn init failed: ", err)
+				return
+			}
+
+			var (
+				lastPrintTime   = time.Now()
+				sendPacketCount uint64
+				recvPacketCount uint64
+			)
+
+			for {
+				recv := reader.ReadPacket(binary.SplitByUint32LE)
+				if reader.Error() != nil {
+					log.Print("receive failed: ", reader.Error())
+					return
+				}
+				recvPacketCount += 1
+
+				writer.WritePacket(recv, binary.SplitByUint32LE)
+				if writer.Error() != nil {
+					log.Print("send failed: ", writer.Error())
+					return
+				}
+				sendPacketCount += 1
+
+				if time.Since(lastPrintTime) > time.Second*2 {
+					lastPrintTime = time.Now()
+					log.Print("server: ", recvPacketCount, sendPacketCount)
+				}
+			}
+		}()
+	}
+}
+
+// Do DH64 key exchange and return a RC4 reader.
+func conn_init(conn net.Conn) (*binary.Writer, *binary.Reader, error) {
+	var (
+		writer = binary.NewWriter(conn)
+		reader = binary.NewReader(conn)
+	)
+
+	rand.Seed(time.Now().UnixNano())
+
+	privateKey, publicKey := dh64.KeyPair()
+	log.Print("server public key: ", publicKey)
+
+	writer.WriteUint64LE(publicKey)
+	if writer.Error() != nil {
+		return nil, nil, writer.Error()
+	}
+	clientPublicKey := reader.ReadUint64LE()
+	if reader.Error() != nil {
+		return nil, nil, reader.Error()
+	}
+	log.Print("client public key: ", clientPublicKey)
+
+	secert := dh64.Secret(privateKey, clientPublicKey)
+	log.Print("secert: ", secert)
+
+	key := make([]byte, 8)
+	binary.PutUint64LE(key, secert)
+	rc4stream, err := rc4.NewCipher(key)
+	if err != nil {
+		return nil, nil, err
+	}
+	log.Print("key: ", hex.EncodeToString(key))
+
+	reader = binary.NewReader(cipher.StreamReader{
+		R: conn,
+		S: rc4stream,
+	})
+	return writer, reader, nil
+}

package/rubyretriever/.rspec

@@ -0,0 +1,2 @@
+--color
+--format progress

package/rubyretriever/.travis.yml

@@ -0,0 +1,7 @@
+language: ruby
+rvm:
+  - 2.0.0
+cache: bundler
+before_install: gem install rspec
+before_install: gem install rake
+script: bundle exec rake

package/rubyretriever/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

package/rubyretriever/Gemfile.lock

@@ -0,0 +1,64 @@
+PATH
+  remote: .
+  specs:
+    rubyretriever (1.4.5)
+      addressable
+      bloomfilter-rb
+      em-http-request
+      em-synchrony
+      htmlentities
+      nokogiri
+      ruby-progressbar
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    addressable (2.4.0)
+    bloomfilter-rb (2.1.1)
+      redis
+    coderay (1.1.0)
+    cookiejar (0.3.0)
+    diff-lcs (1.2.5)
+    em-http-request (1.1.3)
+      addressable (>= 2.3.4)
+      cookiejar (<= 0.3.0)
+      em-socksify (>= 0.3)
+      eventmachine (>= 1.0.3)
+      http_parser.rb (>= 0.6.0)
+    em-socksify (0.3.1)
+      eventmachine (>= 1.0.0.beta.4)
+    em-synchrony (1.0.4)
+      eventmachine (>= 1.0.0.beta.1)
+    eventmachine (1.2.0.1)
+    htmlentities (4.3.4)
+    http_parser.rb (0.6.0)
+    method_source (0.8.2)
+    mini_portile2 (2.0.0)
+    nokogiri (1.6.7.2)
+      mini_portile2 (~> 2.0.0.rc2)
+    pry (0.10.1)
+      coderay (~> 1.1.0)
+      method_source (~> 0.8.1)
+      slop (~> 3.4)
+    rake (10.3.2)
+    redis (3.2.2)
+    rspec (2.99.0)
+      rspec-core (~> 2.99.0)
+      rspec-expectations (~> 2.99.0)
+      rspec-mocks (~> 2.99.0)
+    rspec-core (2.99.0)
+    rspec-expectations (2.99.0)
+      diff-lcs (>= 1.1.3, < 2.0)
+    rspec-mocks (2.99.0)
+    ruby-progressbar (1.7.5)
+    slop (3.6.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.6)
+  pry
+  rake (~> 10.3)
+  rspec (~> 2.14)
+  rubyretriever!

package/rubyretriever/LICENSE

@@ -0,0 +1,20 @@
+2016 (c) Joseph Michael Norton - @JoeNorton - http://Norton.io
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

package/rubyretriever/Rakefile

@@ -0,0 +1,7 @@
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec) do |task|
+  task.rspec_opts = %w(--color --format d)
+end
+
+task default: :spec

package/rubyretriever/bin/rr

@@ -0,0 +1,79 @@
+#! /usr/bin/env ruby
+require 'retriever'
+require 'optparse'
+
+options = {}
+optparse = OptionParser.new do |opts|
+  # Set a banner, displayed at the top
+  # of the help screen.
+  opts.banner = 'Usage: rr [MODE FLAG] [options] Target_URL'
+  options['sitemap'] = false
+  opts.on('-s', '--sitemap [FORMAT]', 'MODE FLAG: Sitemap mode') do |type|
+    options['sitemap'] = type || ''
+  end
+  options['fileharvest'] = false
+  opts.on('-f', '--files FILETYPE', 'MODE FLAG: Fileharvest mode') do |file_e|
+    options['fileharvest'] = file_e
+  end
+  options['seo'] = false
+  opts.on('-e', '--seo', 'MODE FLAG: SEO mode') do
+    options['seo'] = true
+  end
+  options['filename'] = nil
+  opts.on('-o', '--out FILENAME', 'Dump output to file') do |file|
+    options['filename'] = file
+  end
+  # Define the options, and what they do
+  options['verbose'] = false
+  opts.on('-v', '--verbose', 'Output more information') do
+    options['verbose'] = true
+  end
+  options['progress'] = false
+  opts.on('-p', '--progress', 'Output progress bar') do
+    options['progress'] = true
+  end
+  options['maxpages'] = 100
+  opts.on('-l',
+          '--limit PAGE_LIMIT_#',
+          'set a max on the total number of crawled pages') do |maxp|
+    options['maxpages'] = maxp
+  end
+  options['autodown'] = false
+  opts.on('-a', '--auto', 'Automatically download all files located') do
+    options['autodown'] = true
+  end
+  # This displays the help screen, all programs are
+  # assumed to have this option.
+  opts.on('-h', '--help', 'Display this screen') do
+    puts opts
+    exit
+  end
+end
+
+optparse.parse!
+if ARGV[0].nil?
+  abort("###Missing Required Argument\nUsage: rr [mode] [options] Target_URL")
+end
+
+ARGV.each do|q|
+  if options['verbose']
+    puts '###############################'
+    puts '### [RubyRetriever]'
+    puts '### Creating Sitemap' if options['sitemap']
+    puts "### Outputting in format: #{options['sitemap']}" if options['sitemap']
+    puts '### Performing File Harvest' if options['fileharvest']
+    if options['fileharvest']
+      puts "### Searching for filetype: #{options['fileharvest']}"
+    end
+    puts '### Performing SEO Scrape' if options['seo']
+    puts "### Writing to file: #{options['filename']}" if options['filename']
+    puts '### Being verbose'
+    puts "### Stopping after #{options['maxpages']} pages"
+  end
+  puts '###############################'
+  puts "### [RubyRetriever] go fetch #{q}"
+  Retriever::CLI.new(q, options)
+  puts '### [RubyRetriever] is done.'
+  puts '###############################'
+  puts
+end

package/rubyretriever/lib/retriever/cli.rb

@@ -0,0 +1,25 @@
+module Retriever
+  #
+  class CLI
+    def initialize(url, options)
+      # kick off the fetch mode of choice
+      @fetch = choose_fetch_mode(url, options)
+      @fetch.dump
+      @fetch.write if options['filename']
+      @fetch.autodownload if options['autodown'] && options['fileharvest']
+      @fetch.gen_xml if /XML/i =~ options['sitemap'].to_s
+    end
+
+    def choose_fetch_mode(url, options)
+      if options['fileharvest']
+        Retriever::FetchFiles.new(url, options)
+      elsif options['sitemap']
+        Retriever::FetchSitemap.new(url, options)
+      elsif options['seo']
+        Retriever::FetchSEO.new(url, options)
+      else
+        fail '### Error: No Mode Selected'
+      end
+    end
+  end
+end

package/rubyretriever/lib/retriever/core_ext.rb

@@ -0,0 +1,13 @@
+require 'htmlentities'
+#
+module SourceString
+  refine String do
+    def decode_html
+      HTMLEntities.new.decode(self)
+    end
+
+    def encode_utf8_and_replace
+      encode('UTF-8', invalid: :replace, undef: :replace)
+    end
+  end
+end