Skip to content

Updating CDH configuration using python and REST APIs

by on May 26, 2015

I have improved my previous script in the previous post

Now I’ve composed some python script (my first serious python script…) (And I think it can be done easily in bash…)

The script could be beautified, but I think it is quiet readable as it is now.

this particular script modify two parameters for YARN.

the parameters and their values are hard-coded, but it seems not too complicated to get everything from the input as variable.

Next step – creating CDH clusters using scripts only.

modify_yarn_config.py

#!/usr/bin/python
import urllib2
import sys, getopt
import base64
from urlparse import urlparse
import json
from pprint import pprint
import cdhRest
yarn_json = ' { "items" : [ { "name" : "yarn_nodemanager_resource_cpu_vcores", "value" : "8" }, { "name" : "yarn_nodemanager_resource_memory_mb", "value" : "8192" } ] }'
content_header = {'Content-type':'application/json', 'Accept':'application/vnd.error+json,application/json', 'Accept-Version':'1.0'}
yarnJsonObj = json.loads(yarn_json)
def main(argv):
  chost = ''
  username = ''
  password = ''
  try:
    opts, args = getopt.getopt(argv,"h:u:p:",["chost=","user=","pass="])
  except getopt.GetoptError:
    print 'test.py -h <chost> -u <username> -p <password>'
    sys.exit(2)
  for opt, arg in opts:
    if opt in ("-h", "--chost"):
      chost = arg
    elif opt in ("-u", "--user"):
      username = arg
    elif opt in ("-p", "--pass"):
      password = arg

  base64string = base64.encodestring( '%s:%s' % (username, password))[:-1]
  authheader = 'Basic %s' % base64string
  apiver = cdhRest.getversion(chost, authheader)
  baseurl = "http://" + chost + ":7180/api/" + apiver + "/clusters"
  clusterslist = cdhRest.get_cluster_names(baseurl, authheader)
  for cluster in clusterslist:
    baseurl1 = baseurl + "/" + cluster + "/services"
    service = cdhRest.get_service_name_by_type(baseurl1, "YARN", authheader)
    baseurl1 = baseurl1 + "/" + service + "/roleConfigGroups"
    confgroups = cdhRest.get_conf_groups(baseurl1, "NODEMANAGER", authheader)
    for confgroup in confgroups:
      baseurl2 = baseurl1 + "/" + confgroup + "/config?view=full"
      req = urllib2.Request(baseurl2)
      req.add_header("Authorization", authheader)
      handle = urllib2.urlopen(req)
      thepage = handle.read()
      data = json.loads(thepage)
# example taken from here http://stackoverflow.com/questions/21243834/doing-put-using-python-urllib2
      baseURL = baseurl1 + "/" + confgroup + "/config"
      request = urllib2.Request(url=baseURL, data=json.dumps(yarnJsonObj), headers=content_header)
      request.add_header("Authorization", authheader)
      request.get_method = lambda: 'PUT' #if I remove this line then the POST works fine.

      response = urllib2.urlopen(request)

if __name__ == "__main__":
  main(sys.argv[1:])

cdhRest.py

#!/usr/bin/python

import urllib2
import sys, getopt
import base64
from urlparse import urlparse
import json
from pprint import pprint

#
# function get version - get the REST api version of cm
#
def getversion(chost, authheader):
  theurl = "http://" + chost + ":7180/api/version"
  req = urllib2.Request(theurl)
  req.add_header("Authorization", authheader)
  handle = urllib2.urlopen(req)
  ret = handle.read()
  return ret

#
# get_cluster_names - get list of cluster names
#
def get_cluster_names(theurl, authheader):
  req = urllib2.Request(theurl)
  req.add_header("Authorization", authheader)
  handle = urllib2.urlopen(req)
  thepage = handle.read()
  data = json.loads(thepage)
  ret = []
  for xx in data["items"]:
    ret = ret + [xx["name"]]
  return ret

#
# get_service_name_by_type - get service name by type
#
def get_service_name_by_type(theurl, type, authheader):
  req = urllib2.Request(theurl)
  req.add_header("Authorization", authheader)
  handle = urllib2.urlopen(req)
  thepage = handle.read()
  data = json.loads(thepage)
  for xx in data["items"]:
    aa = xx["type"]
    if (aa == type):
    ret = xx["name"]
    return ret

#
# get_conf_groups - get list of configuration groups
#
def get_conf_groups(theurl, roleType, authheader):
  req = urllib2.Request(theurl)
  req.add_header("Authorization", authheader)
  handle = urllib2.urlopen(req)
  thepage = handle.read()
  data = json.loads(thepage)
  ret = []
  for xx in data["items"]:
    if (xx["roleType"] == roleType):
      ret = ret + [xx["name"]]
  return ret

aaa

Advertisements

Leave a Reply

Fill in your details below or click an icon to log in:

WordPress.com Logo

You are commenting using your WordPress.com account. Log Out / Change )

Twitter picture

You are commenting using your Twitter account. Log Out / Change )

Facebook photo

You are commenting using your Facebook account. Log Out / Change )

Google+ photo

You are commenting using your Google+ account. Log Out / Change )

Connecting to %s

%d bloggers like this: