Browse Source

Add new files

kouze 2 years ago
parent
commit
a4a3898c2d

+ 9 - 0
decodeCFemail.php

@ -0,0 +1,9 @@
1
<?php
2
echo deCFEmail($argv[1]);
3
 
4
function deCFEmail($c){
5
   $k = hexdec(substr($c,0,2));
6
   for($i=2,$m='';$i<strlen($c)-1;$i+=2)$m.=chr(hexdec(substr($c,$i,2))^$k);
7
   return $m;
8
}
9
?>

+ 2 - 0
delete_failed.sh

@ -0,0 +1,2 @@
1
#! /bin/bash
2
rtcontrol -D rtorrent_rc=/home/rtorrent/.rtorrent.rc loaded=+1h message="*Torrent Inconnu*","*nregistered torrent*","*Torrent is not authorized for use on this tracker*" --cull --yes  -q --detach

+ 80 - 0
delete_torrent.sh

@ -0,0 +1,80 @@
1
#!/bin/bash
2
TORRENTHASH=$1
3
RTORRENTRC=$2
4
TRACKER=$3
5

													
6
function deleteTorrent() {
7
  if [[ $TORRENTHASH != "" ]]; then
8
    RCODE=$($RTXMLRPC d.tracker_announce $TORRENTHASH)
9
    $RTCONTROL hash="$TORRENTHASH" --cull --yes -q --detach 
10
  fi
11
}
12

													
13
function deleteTorrent_old() {
14
  if [[ $TORRENTHASH != "" ]]; then
15
    # get the full file name with directory
16
    base_path=$($RTXMLRPC d.get_base_path $TORRENTHASH)
17

													
18
    #update tracker
19
    RCODE=$($RTXMLRPC d.tracker_announce $TORRENTHASH)
20

													
21
    #sleep 1m
22

													
23
    # delete torrent from rTorrent
24
    RCODE=$($RTXMLRPC d.stop $TORRENTHASH)
25
    RCODE=$($RTXMLRPC d.close $TORRENTHASH)
26
    RCODE=$($RTXMLRPC d.erase $TORRENTHASH)
27

													
28
    # delete file or directory from the system
29
    # check if file/directory exists
30
    if [ -e "$base_path" ]; then
31
      # test if it's a file
32
      if [ -f "$base_path" ]; then
33
        # delete the file
34
        rm "$base_path"
35
      else
36
        # delete the directory
37
        rm -r "$base_path"
38
      fi
39
    fi
40
    echo "delete_torrent.sh: Torrent $base_path deleted"
41
  fi
42
}
43

													
44

													
45

													
46
function checkSnatched()
47
{
48
  URL="https://hodor.ismy.guru/api/torrent/$TORRENTHASH?ak=8dab665cc879285e23ea8ebf816e433e"
49
  JSON=$(curl "$URL" 2>/dev/null)
50
  SNATCHED=$(echo $JSON | jq -r .snatched)
51
  TORRENTID=$(echo $JSON | jq -r .id)
52
  if [[ "$SNATCHED" == "null" ]]; then
53
    sleep 5s
54
    echo "delete_torrent.sh: updating tracker completed for $TORRENTHASH id $TORRENTID"
55
    URL="https://hodor.ismy.guru/api/torrent/$TORRENTID?ak=8dab665cc879285e23ea8ebf816e433e"
56
    JSON=$(curl -X PUT "$URL" 2>/dev/null)
57
    CONTAINSNATCHED=$(echo $JSON | grep ".snatched")
58
    SNATCHED=$(echo $JSON | jq -r .snatched)
59
    if [[ "$CONTAINSNATCHED" == "" || "$SNATCHED" == "null" ]]; then
60
      echo "$TORRENTHASH ¤¤ $TORRENTID" >> /home/nykouze/scripts/delete_torrent.failed
61
      echo "delete_torrent.sh: issue during the update at tracker level"
62
    else
63
      echo "delete_torrent.sh: completed update successfull for $TORRENTHASH id $TORRENTID"
64
    fi
65
  fi
66
}
67

													
68
if [[ "$RTORRENTRC" != "" ]]; then
69
  RTORRENTRC="/home/rtorrent/.$RTORRENTRC"
70
  RTXMLRPC="/home/nykouze/bin/rtxmlrpc -D rtorrent_rc=$RTORRENTRC"
71
  RTCONTROL="/home/nykouze/bin/rtcontrol -D rtorrent_rc=$RTORRENTRC"
72
  RTORRENTID=$(pgrep -f $RTORRENTRC)
73
  if [[ $RTORRENTID != "" ]]; then
74
    if [[ "$TRACKER" == "gandalf.ismy.guru" ]]; then
75
      echo "delete_torrent.sh: Hodor process for $TORRENTHASH"
76
      checkSnatched
77
    fi
78
    deleteTorrent
79
  fi
80
fi

+ 231 - 0
newznab/misc/update_scripts/import2.php

@ -0,0 +1,231 @@
1
<?php
2

													
3
define('FS_ROOT', realpath(dirname(__FILE__)));
4
require_once(FS_ROOT."/../../www/config.php");
5
require_once(FS_ROOT."/../../www/lib/framework/db.php");
6
require_once(FS_ROOT."/../../www/lib/releases.php");
7
require_once(FS_ROOT."/../../www/lib/nzbinfo.php");
8
require_once(FS_ROOT."/../../www/lib/nzb.php");
9
require_once(FS_ROOT."/../../www/lib/util.php");
10
require_once(FS_ROOT."/../../www/lib/category.php");
11

													
12
$releases = new Releases();
13
$db = new Db();
14
$cat = new Category();
15
$releaseRegex = new ReleaseRegex();
16
$nzb = new NZB();
17
$page = new Page();
18
$num = 0;
19
$usefilename = true;
20
$dupecheck = true;
21
$movefiles = true;
22
$categoryoverride = -1;
23

													
24
if(empty($argc) || $argc <= 1){
25
    $path = "./";
26
} else {
27
    $path = (!endsWith($argv[1], "/") ? $argv[1]."/" : $argv[1]);
28
    if (isset($argv[2]))
29
        $usefilename = strtolower($argv[2]) == 'true';
30
    if (isset($argv[3]))
31
        $dupecheck = strtolower($argv[3]) == 'true';
32
    if (isset($argv[4]))
33
        $movefiles = strtolower($argv[4]) == 'true';
34
    if (isset($argv[5]) && is_numeric($argv[5]))
35
        $categoryoverride = $argv[5];
36
}
37

													
38
$groups = $db->query("SELECT ID, name FROM groups");
39
foreach ($groups as $group)
40
    $siteGroups[$group["name"]] = $group["ID"];
41

													
42
echo "\nUsage: php import.php [path(string)] [usefilename(true/false)] [dupecheck(true/false)] [movefiles(true/false)] [overridecategory(number)]\n";
43

													
44
$filestoprocess = glob($path."*.{nzb,nzb.gz}", GLOB_BRACE);
45
$items = count($filestoprocess);
46
$matches = array();
47
$digits = preg_match_all( "/[0-9]/", count($filestoprocess), $matches);
48
$dupepath = $path."dupe/";
49
$nogrouppath = $path."nogroup/";
50
$importedpath = $path."imported/";
51
$noregexpath = $path."noregex/";
52
$errorpath = $path."error/";
53
$missinggroups = array ();
54
echo "[Importing ".$items." *.nzb file".($items != 1 ? "s" : "")." from ".$path.($usefilename?" U":" Not u")."sing filename, ".($dupecheck?"C":"Not c")."hecking for duplicates".($categoryoverride != -1 ? ", Forcing category to ".$categoryoverride : "")."]\n\n";
55

													
56
foreach($filestoprocess as $nzbFile)
57
{
58
    $groupID = -1;
59
    $groupName = "";
60
    $num++;
61
    $nzbInfo = new nzbInfo;
62

													
63
    if (!$nzbInfo->loadFromFile($nzbFile, true))
64
    {
65
        echo "Failed to load nzb from disk ".$nzbFile."\n";
66
        if ($movefiles)
67
        {
68
            if (!file_exists($errorpath)) mkdir($errorpath);
69
            if (!file_exists($errorpath.basename($nzbFile))) rename($nzbFile, $errorpath.basename($nzbFile));
70
        }
71
    }
72
    else
73
    {
74
        if ($dupecheck)
75
        {
76
            $dupes = $db->queryOneRow(sprintf("SELECT EXISTS(SELECT 1 FROM releases WHERE gid = %s) as total", $db->escapeString($nzbInfo->gid)));
77
            if ($dupes['total'] > 0)
78
            {
79
                echo sprintf("%0".$digits."d %.2f%% Error : Dupe %s - GID(%s)\n", $items-$num, $num/$items*100, $nzbFile, $nzbInfo->gid);
80
                if ($movefiles)
81
                {
82
                    if (!file_exists($dupepath)) mkdir($dupepath);
83
                    if (!file_exists($dupepath.basename($nzbFile))) rename($nzbFile, $dupepath.basename($nzbFile));
84
                }
85
                continue;
86
            }
87
        }
88

													
89
        foreach($nzbInfo->groups as $group)
90
        {
91
            $group = (string)$group;
92
            if (array_key_exists($group, $siteGroups))
93
            {
94
                $groupID = $siteGroups[$group];
95
                $groupName = $group;
96
                break;
97
            }
98
        }
99

													
100
        if ($groupID == -1)
101
        {
102
            echo sprintf("%0".$digits."d %.2f%% Error : Missing group %s - Group(%s)\n", $items-$num, $num/$items*100, $nzbFile, str_replace("alt.binaries.", "a.b.", implode(",",$nzbInfo->groups)));
103
            if ($movefiles)
104
            {
105
                if (!file_exists($nogrouppath)) mkdir($nogrouppath);
106
                if (!file_exists($nogrouppath.basename($nzbFile))) rename($nzbFile, $nogrouppath.basename($nzbFile));
107
                $missinggroups = array_merge($missinggroups, $nzbInfo->groups);
108
            }
109
            continue;
110
        }
111
        else
112
        {
113
            if ($usefilename)
114
            {
115
                $relguid = md5(uniqid());
116
                //$name = $releases->cleanReleaseName(str_replace(".nzb", "", basename($nzbFile)));
117
                $name = str_replace(".nzb", "", basename($nzbFile));
118
		$catId = ($categoryoverride != -1 ? $categoryoverride : $cat->determineCategory($groupName, $name));
119
		//$catId = $cat->determineCategory($groupName, $name);
120
                $relid = $releases->insertRelease($name, $nzbInfo->filecount, $groupID, $relguid, $catId, "", date("Y-m-d H:i:s", $nzbInfo->postedlast), $nzbInfo->poster, "", $page->site);
121
                $db->exec(sprintf("update releases set totalpart = %d, size = %s, completion = %d, GID=%s where ID = %d",  $nzbInfo->filecount, $nzbInfo->filesize, $nzbInfo->completion, $db->escapeString($nzbInfo->gid), $relid ));
122

													
123
                $nzbfilename = $nzb->getNZBPath($relguid, $page->site->nzbpath, true);
124
                $fp = _gzopen($nzbfilename, "w");
125
                if ($fp)
126
                {
127
                    gzwrite($fp, $nzbInfo->toNzb());
128
                    gzclose($fp);
129
                    echo sprintf("¤¤ %0".$digits."d %.2f%% Imported ¤¤ %s ¤¤ $relid\n", $items-$num, $num/$items*100, $name);
130
                    if ($movefiles)
131
                    {
132
                        if (!file_exists($importedpath)) mkdir($importedpath);
133
                        if (!file_exists($importedpath.basename($nzbFile))) rename($nzbFile, $importedpath.basename($nzbFile));
134
                    }
135
                }
136
                else
137
                {
138
                    echo sprintf("%0".$digits."d %.2f%% Error : Failed to write file to disk %s\n", $items-$num, $num/$items*100, $nzbfilename);
139
                    if ($movefiles)
140
                    {
141
                        if (!file_exists($errorpath)) mkdir($errorpath);
142
                        if (!file_exists($errorpath.basename($nzbFile))) rename($nzbFile, $errorpath.basename($nzbFile));
143
                    }
144
                    $releases->delete($relid);
145
                }
146
            }
147
            else
148
            {
149
                $numbins = 0;
150
                $numparts = 0;
151
                $binaryId = 0;
152
                $groupRegexes = $releaseRegex->getForGroup($groupName);
153
                foreach($nzbInfo->nzb as $postFile)
154
                {
155
                    $regexMatches = array();
156

													
157
                    foreach($groupRegexes as $groupRegex)
158
                    {
159
                        $regexCheck = $releaseRegex->performMatch($groupRegex, $postFile["subject"]);
160
                        if ($regexCheck !== false)
161
                        {
162
                            $regexMatches = $regexCheck;
163
                            break;
164
                        }
165
                    }
166

													
167
                    if (!empty($regexMatches))
168
                    {
169
                        $relparts = explode("/", $regexMatches['parts']);
170
                        $regexMatches['regcatid'] = ($categoryoverride != -1 ? $categoryoverride : $regexMatches['regcatid']);
171

													
172
                        $sql = sprintf("INSERT INTO binaries (name, fromname, date, xref, totalParts, groupID, binaryhash, dateadded,
173
                        categoryID, regexID, reqID, procstat, relpart, reltotalpart, relname)
174
                        values (%s, %s, %s, %s, %d, %d, %s, NOW(), %s, %d, %s, %d, %d, %d, %s )",
175
                            $db->escapeString($postFile["subject"]), $db->escapeString($postFile["poster"]),
176
                            $db->escapeString(date("Y-m-d H:i:s", $postFile["posted"])),
177
                            $db->escapeString(implode(': ', $nzbInfo->groups).':'),
178
                            $postFile["segmenttotal"], $groupID,
179
                            $db->escapeString(md5($postFile["subject"].$postFile["poster"].$groupID)),
180
                            $regexMatches['regcatid'],
181
                            $regexMatches['regexID'], $db->escapeString($regexMatches['reqID']),
182
                            Releases::PROCSTAT_TITLEMATCHED, $relparts[0], $relparts[1], $db->escapeString(str_replace('_',' ', $regexMatches['name']))
183
                        );
184
                        $binaryId = $db->queryInsert($sql);
185
                        $numbins++;
186

													
187
                        if (count($postFile['segments']) > 0)
188
                        {
189
                            $sql = "INSERT INTO parts (binaryID, messageID, number, partnumber, size) values ";
190
                            foreach($postFile['segments'] as $fileSegmentNum => $fileSegment)
191
                            {
192
                                $sql .= sprintf("(%d, %s, 0, %d, %d),", $binaryId, $db->escapeString($fileSegment), $fileSegmentNum, $postFile['segmentbytes'][$fileSegmentNum]);
193
                                $numparts++;
194
                            }
195
                            $db->queryInsert(substr($sql, 0, -1));
196
                        }
197
                    }
198
                }
199
                if ($binaryId != 0)
200
                {
201
                    echo sprintf("%0".$digits."d %.2f%% Imported %s (%d:%s-%d/%d)\n", $items-$num, $num/$items*100, basename($nzbFile), $regexMatches['regcatid'], $regexMatches['regexID'], $numbins, $numparts);
202
                    if ($movefiles)
203
                    {
204
                        if (!file_exists($importedpath)) mkdir($importedpath);
205
                        if (!file_exists($importedpath.basename($nzbFile))) rename($nzbFile, $importedpath.basename($nzbFile));
206
                    }
207
                }
208
                else
209
                {
210
                    echo sprintf("%0".$digits."d %.2f%% Error : No Regex Match %s\n", $items-$num, $num/$items*100, basename($nzbFile));
211
                    if ($movefiles)
212
                    {
213
                        if (!file_exists($noregexpath)) mkdir($noregexpath);
214
                        if (!file_exists($noregexpath.basename($nzbFile))) rename($nzbFile, $noregexpath.basename($nzbFile));
215
                    }
216
                }
217
            }
218
        }
219
    }
220
}
221

													
222
if (count($missinggroups) > 0)
223
{
224
    $missinggroups = array_unique($missinggroups);
225
    $grpsql = "INSERT INTO groups (name, backfill_target, first_record, first_record_postdate, last_record, last_record_postdate, last_updated, minfilestoformrelease, minsizetoformrelease, active, regexmatchonly, description) VALUES ('%s', 0, 0, null, 0, null, null, null, null, 0, 1, 'Added by import');\n";
226
    $grpout = "";
227
    foreach ($missinggroups as $mg)
228
        $grpout .= sprintf($grpsql, $mg);
229

													
230
    @file_put_contents(sprintf("missing_groups_%s.sql", uniqid()), $grpout);
231
}

+ 37 - 0
newznab/misc/update_scripts/import_nfo.php

@ -0,0 +1,37 @@
1
<?php
2

													
3
define('FS_ROOT', realpath(dirname(__FILE__)));
4
require_once(FS_ROOT."/../../www/config.php");
5
require_once(FS_ROOT."/../../www/lib/framework/db.php");
6

													
7
$db = new Db();
8

													
9
if(empty($argc) || $argc <= 2){
10
    exit;
11
} else {
12
    $releaseid = $argv[1];
13
    $nfofile = $argv[2];
14

													
15
    $blob = file_get_contents($nfofile);
16
    //echo printf($blob);
17

													
18
    $query = sprintf("SELECT id FROM releases WHERE ID = %d", $releaseid);
19
    $uid = $db->queryOneRow($query);
20

													
21
    if(!$uid){
22
        echo sprintf("Release ID %d not found in releases", $releaseid);
23
    } else {
24
        $query = sprintf(
25
            "REPLACE INTO releasenfo (ID, releaseID, binaryID, nfo) ".
26
            "VALUES (NULL, %d, 0, compress(%s));",
27
            $releaseid, $db->escapeString($blob));
28
        $id = $db->queryInsert($query);
29
        if(!$id){
30
            echo sprintf("issue when inserting data into releasenfo for id %d", $uid);
31
        }else{
32
            $query = sprintf("UPDATE releases SET releasenfoID = %d WHERE ID = %d", $id, $uid);
33
            $res = $db->exec($query);
34
            //echo sprintf("NFO correctly added for release %d: %s", $releaseid, $res);
35
        }
36
    }
37
}

+ 101 - 0
processNg.sh

@ -0,0 +1,101 @@
1
#!/bin/bash
2
RTORRENTFILE="$HOME/scripts/outputRtorrent.txt"
3
MYFILE="$HOME/scripts/toProcess.txt"
4
BACKUP="$HOME/scripts/oldProcess.txt"
5
ALLBACKUPS="$HOME/scripts/allProcess.txt"
6
NEWZNAB_PATH="/home/newznab/nnplus/misc/update_scripts"
7
FAILEDFILE="$HOME/scripts/outputRtorrent_failed.txt"
8
PAUSEFILE="/tmp/stopNGprocess"
9
START=$(date +%s)
10

													
11
trap ctrl_c INT
12
function ctrl_c() {
13
    exit 0;
14
}
15

													
16
while true; do
17
  END=$(date +%s)
18
  DIFF=$((END-START))
19
  if [ $DIFF -ge 600 ]; then
20
    echo "Deleting failed torrent"
21
    $HOME/scripts/delete_failed.sh
22
    START=$END
23
  fi
24
  if [ ! -e $MYFILE ]; then
25
    rm -f $BACKUP
26
    
27
    if [ -e $FAILEDFILE ]; then
28
      DATE=$(date)
29
      echo $DATE
30
      echo "Reprocessing failed getInfo torrent"
31
      cp $FAILEDFILE $MYFILE && rm $FAILEDFILE
32
      old_IFS=$IFS
33
      IFS=$'\n'
34
      for line in $(cat $MYFILE); do
35
        PARAM1=$(echo $line | awk -F" ¤¤ " '{ print $1 }')
36
        #echo "File:" $PARAM1
37
        PARAM2=$(echo $line | awk -F" ¤¤ " '{ print $2 }')
38
        #echo "Text:" $PARAM2
39
        PARAM3=$(echo $line | awk -F" ¤¤ " '{ print $3 }')
40
        #echo "Text:" $PARAM3
41
        PARAM4=$(echo $line | awk -F" ¤¤ " '{ print $4 }')
42
        #echo "Text:" $PARAM4
43
        PARAM5=$(echo $line | awk -F" ¤¤ " '{ print $5 }')
44
        #echo "Text:" $PARAM5
45
        PARAM6=$(echo $line | awk -F" ¤¤ " '{ print $6 }')
46
        #echo "Text:" $PARAM6
47
        while [[ -e $PAUSEFILE ]]; do
48
          echo "NG Process Paused. Run continueNGprocess to continue"
49
          sleep 10s
50
        done
51
        $HOME/scripts/sendng.sh $PARAM1 $PARAM2 $PARAM3 $PARAM4 $PARAM5 $PARAM6
52
        echo "$line" >> $BACKUP
53
        echo "$(tail -n +2 $MYFILE)" > $MYFILE
54
      done
55
      IFS=$old_IFS
56
      rm -f $MYFILE
57
      cd $NEWZNAB_PATH
58
      echo "Newznab Update Releases"
59
      RETURN=$(/usr/bin/php5 $NEWZNAB_PATH/update_releases.php)
60
    fi
61

													
62
    if [ -e $RTORRENTFILE ]; then
63
      DATE=$(date)
64
      echo $DATE
65
      echo "Processing new files"
66
      cp $RTORRENTFILE $MYFILE && rm $RTORRENTFILE
67
      old_IFS=$IFS
68
      IFS=$'\n'
69
      for line in $(cat $MYFILE); do
70
        PARAM1=$(echo $line | awk -F" ¤¤ " '{ print $1 }')
71
        #echo "File:" $PARAM1
72
        PARAM2=$(echo $line | awk -F" ¤¤ " '{ print $2 }')
73
        #echo "Text:" $PARAM2
74
        PARAM3=$(echo $line | awk -F" ¤¤ " '{ print $3 }')
75
        #echo "Text:" $PARAM3
76
        PARAM4=$(echo $line | awk -F" ¤¤ " '{ print $4 }')
77
        #echo "Text:" $PARAM4
78
        PARAM5=$(echo $line | awk -F" ¤¤ " '{ print $5 }')
79
        #echo "Text:" $PARAM5
80
        PARAM6=$(echo $line | awk -F" ¤¤ " '{ print $6 }')
81
        while [[ -e $PAUSEFILE ]]; do
82
          echo "NG Process Paused. Run continueNGprocess to continue"
83
          sleep 10s
84
        done
85
        $HOME/scripts/sendng.sh $PARAM1 $PARAM2 $PARAM3 $PARAM4 $PARAM5 $PARAM6
86
        echo "$line" >> $ALLBACKUPS
87
        echo "$line" >> $BACKUP
88
        echo "$(tail -n +2 $MYFILE)" > $MYFILE
89
      done
90
      IFS=$old_IFS
91
      rm -f $MYFILE
92
      cd $NEWZNAB_PATH
93
      echo "Newznab Update Releases"
94
      RETURN=$(/usr/bin/php5 $NEWZNAB_PATH/update_releases.php)
95
    fi
96
  fi
97
  if [ ! -e $RTORRENTFILE ]; then
98
#    echo -n "*"
99
    sleep 1m
100
  fi
101
done

+ 25 - 0
processNgFile.sh

@ -0,0 +1,25 @@
1
#!/bin/bash
2
NEWZNAB_PATH="/home/newznab/nnplus/misc/update_scripts"
3
ALLBACKUPS="$HOME/scripts/allProcess.txt"
4

													
5
old_IFS=$IFS
6
IFS=$'\n'
7
for line in $(cat "$1"); do
8
  PARAM1=$(echo $line | awk -F" ¤¤ " '{ print $1 }')
9
  echo "File:" $PARAM1
10
  PARAM2=$(echo $line | awk -F" ¤¤ " '{ print $2 }')
11
  echo "Text:" $PARAM2
12
  PARAM3=$(echo $line | awk -F" ¤¤ " '{ print $3 }')
13
  echo "Text:" $PARAM3
14
  PARAM4=$(echo $line | awk -F" ¤¤ " '{ print $4 }')
15
  echo "Text:" $PARAM4
16
  PARAM5=$(echo $line | awk -F" ¤¤ " '{ print $5 }')
17
  echo "Text:" $PARAM5
18
  PARAM6=$(echo $line | awk -F" ¤¤ " '{ print $6 }')
19
  echo "Text:" $PARAM6
20
  $HOME/scripts/sendng.sh $PARAM1 $PARAM2 $PARAM3 $PARAM4 $PARAM5 $PARAM6
21
  echo "$line" >> $ALLBACKUPS
22
done
23
IFS=$old_IFS
24
cd $NEWZNAB_PATH
25
/usr/bin/php5 $NEWZNAB_PATH/update_releases.php